{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from graphviz import Digraph\n",
    "import math\n",
    "import torch\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -*- coding: UTF-8 -*-\n",
    "'''\n",
    "此脚本用于定义Scalar类，以及相应的可视化工具\n",
    "'''\n",
    "\n",
    "\n",
    "from graphviz import Digraph\n",
    "import math\n",
    "\n",
    "\n",
    "class Scalar:\n",
    "    \n",
    "    def __init__(self, value, prevs=[], op=None, label='', requires_grad=True):\n",
    "        # 节点的值\n",
    "        self.value = value\n",
    "        # 节点的标识（label）和对应的运算（op），用于作图\n",
    "        self.label = label\n",
    "        self.op = op\n",
    "        # 节点的前节点，即当前节点是运算的结果，而前节点是参与运算的量\n",
    "        self.prevs = prevs\n",
    "        # 是否需要计算该节点偏导数，即∂loss/∂self（loss表示最后的模型损失）\n",
    "        self.requires_grad = requires_grad\n",
    "        # 该节点偏导数，即∂loss/∂self\n",
    "        self.grad = 0.0\n",
    "        # 如果该节点的prevs非空，存储所有的∂self/∂prev\n",
    "        self.grad_wrt = dict()\n",
    "        # 作图需要，实际上对计算没有作用\n",
    "        self.back_prop = dict()\n",
    "        \n",
    "    def __repr__(self):\n",
    "        return f'Scalar(value={self.value:.2f}, grad={self.grad:.2f})'\n",
    "    \n",
    "    def __add__(self, other):\n",
    "        '''\n",
    "        定义加法，self + other将触发该函数\n",
    "        '''\n",
    "        if not isinstance(other, Scalar):\n",
    "            other = Scalar(other, requires_grad=False)\n",
    "        # output = self + other\n",
    "        output = Scalar(self.value + other.value, [self, other], '+')\n",
    "        output.requires_grad = self.requires_grad or other.requires_grad\n",
    "        # 计算偏导数 ∂output/∂self = 1\n",
    "        output.grad_wrt[self] = 1\n",
    "        # 计算偏导数 ∂output/∂other = 1\n",
    "        output.grad_wrt[other] = 1\n",
    "        return output\n",
    "    \n",
    "    def __sub__(self, other):\n",
    "        '''\n",
    "        定义减法，self - other将触发该函数\n",
    "        '''\n",
    "        if not isinstance(other, Scalar):\n",
    "            other = Scalar(other, requires_grad=False)\n",
    "        # output = self - other\n",
    "        output = Scalar(self.value - other.value, [self, other], '-')\n",
    "        output.requires_grad = self.requires_grad or other.requires_grad\n",
    "        # 计算偏导数 ∂output/∂self = 1\n",
    "        output.grad_wrt[self] = 1\n",
    "        # 计算偏导数 ∂output/∂other = -1\n",
    "        output.grad_wrt[other] = -1\n",
    "        return output\n",
    "    \n",
    "    def __mul__(self, other):\n",
    "        '''\n",
    "        定义乘法，self * other将触发该函数\n",
    "        '''\n",
    "        if not isinstance(other, Scalar):\n",
    "            other = Scalar(other, requires_grad=False)\n",
    "        # output = self * other\n",
    "        output = Scalar(self.value * other.value, [self, other], '*')\n",
    "        output.requires_grad = self.requires_grad or other.requires_grad\n",
    "        # 计算偏导数 ∂output/∂self = other\n",
    "        output.grad_wrt[self] = other.value\n",
    "        # 计算偏导数 ∂output/∂other = self\n",
    "        output.grad_wrt[other] = self.value\n",
    "        return output\n",
    "    \n",
    "    def __pow__(self, other):\n",
    "        '''\n",
    "        定义乘方，self**other将触发该函数\n",
    "        '''\n",
    "        assert isinstance(other, (int, float))\n",
    "        # output = self ** other\n",
    "        output = Scalar(self.value ** other, [self], f'^{other}')\n",
    "        output.requires_grad = self.requires_grad\n",
    "        # 计算偏导数 ∂output/∂self = other * self**(other-1)\n",
    "        output.grad_wrt[self] = other * self.value**(other - 1)\n",
    "        return output\n",
    "    \n",
    "    def sigmoid(self):\n",
    "        '''\n",
    "        定义sigmoid\n",
    "        '''\n",
    "        s = 1 / (1 + math.exp(-1 * self.value))\n",
    "        output = Scalar(s, [self], 'sigmoid')\n",
    "        output.requires_grad = self.requires_grad\n",
    "        # 计算偏导数 ∂output/∂self = output * (1 - output)\n",
    "        output.grad_wrt[self] = s * (1 - s)\n",
    "        return output\n",
    "    \n",
    "    def __rsub__(self, other):\n",
    "        '''\n",
    "        定义右减法，other - self将触发该函数\n",
    "        '''\n",
    "        if not isinstance(other, Scalar):\n",
    "            other = Scalar(other, requires_grad=False)\n",
    "        output = Scalar(other.value - self.value, [self, other], '-')\n",
    "        output.requires_grad = self.requires_grad or other.requires_grad\n",
    "        # 计算偏导数 ∂output/∂self = -1\n",
    "        output.grad_wrt[self] = -1\n",
    "        # 计算偏导数 ∂output/∂other = 1\n",
    "        output.grad_wrt[other] = 1\n",
    "        return output\n",
    "    \n",
    "    def __radd__(self, other):\n",
    "        '''\n",
    "        定义右加法，other + self将触发该函数\n",
    "        '''\n",
    "        return self.__add__(other)\n",
    "    \n",
    "    def __rmul__(self, other):\n",
    "        '''\n",
    "        定义右乘法，other * self将触发该函数\n",
    "        '''\n",
    "        return self * other\n",
    "    \n",
    "    def backward(self, fn=None):\n",
    "        '''\n",
    "        由当前节点出发，求解以当前节点为顶点的计算图中每个节点的偏导数，i.e. ∂self/∂node\n",
    "        参数\n",
    "        ----\n",
    "        fn ：画图函数，如果该变量不等于None，则会返回向后传播每一步的计算的记录\n",
    "        返回\n",
    "        ----\n",
    "        re ：向后传播每一步的计算的记录\n",
    "        '''\n",
    "        def _topological_order():\n",
    "            '''\n",
    "            利用深度优先算法，返回计算图的拓扑排序（topological sorting）\n",
    "            '''\n",
    "            def _add_prevs(node):\n",
    "                if node not in visited:\n",
    "                    visited.add(node)\n",
    "                    for prev in node.prevs:\n",
    "                        _add_prevs(prev)\n",
    "                    ordered.append(node)\n",
    "            ordered, visited = [], set()\n",
    "            _add_prevs(self)\n",
    "            return ordered\n",
    "\n",
    "        def _compute_grad_of_prevs(node):\n",
    "            '''\n",
    "            由node节点出发，向后传播\n",
    "            '''\n",
    "            # 作图需要，实际上对计算没有作用\n",
    "            node.back_prop = dict()\n",
    "            # 得到当前节点在计算图中的梯度。由于一个节点可以在多个计算图中出现，\n",
    "            # 使用cg_grad记录当前计算图的梯度\n",
    "            dnode = cg_grad[node]\n",
    "            # 使用node.grad记录节点的累积梯度\n",
    "            node.grad += dnode\n",
    "            for prev in node.prevs:\n",
    "                # 由于node节点的偏导数已经计算完成，可以向后扩散（反向传播）\n",
    "                # 需要注意的是，向后扩散到上游节点是累加关系\n",
    "                grad_spread = dnode * node.grad_wrt[prev]\n",
    "                cg_grad[prev] = cg_grad.get(prev, 0.0) + grad_spread\n",
    "                node.back_prop[prev] = node.back_prop.get(prev, 0.0) + grad_spread\n",
    "        \n",
    "        # 当前节点的偏导数等于1，因为∂self/∂self = 1。这是反向传播算法的起点\n",
    "        cg_grad = {self: 1}\n",
    "        # 为了计算每个节点的偏导数，需要使用拓扑排序的倒序来遍历计算图\n",
    "        ordered = reversed(_topological_order())\n",
    "        re = []\n",
    "        for node in ordered:\n",
    "            _compute_grad_of_prevs(node)\n",
    "            # 作图需要，实际上对计算没有作用\n",
    "            if fn is not None:\n",
    "                re.append(fn(self, 'backward'))\n",
    "        return re\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def _get_node_attr(node, direction='forward'):\n",
    "    '''\n",
    "    节点的属性\n",
    "    '''\n",
    "    node_type = _get_node_type(node)\n",
    "    # 设置字体\n",
    "    res = {'fontname': 'Menlo'}\n",
    "    def _forward_attr():\n",
    "        if node_type == 'param':\n",
    "            node_text = f'{{ grad=None | value={node.value:.2f} | {node.label}}}'\n",
    "            res.update(\n",
    "                dict(label=node_text, shape='record', fontsize='10', fillcolor='lightgreen', style='filled, bold'))\n",
    "            return res\n",
    "        elif node_type == 'computation':\n",
    "            node_text = f'{{ grad=None | value={node.value:.2f} | {node.op}}}'\n",
    "            res.update(\n",
    "                dict(label=node_text, shape='record', fontsize='10', fillcolor='gray94', style='filled, rounded'))\n",
    "            return res\n",
    "        elif node_type == 'input':\n",
    "            if node.label == '':\n",
    "                node_text = f'input={node.value:.2f}'\n",
    "            else:\n",
    "                node_text = f'{node.label}={node.value:.2f}'\n",
    "            res.update(dict(label=node_text, shape='oval', fontsize='10'))\n",
    "            return res\n",
    "    \n",
    "    def _backward_attr():\n",
    "        attr = _forward_attr()\n",
    "        attr['label'] = attr['label'].replace('grad=None', f'grad={node.grad:.2f}')\n",
    "        if not node.requires_grad:\n",
    "            attr['style'] = 'dashed'\n",
    "        # 为了作图美观\n",
    "        # 如果向后扩散（反向传播）的梯度等于0，或者扩散给不需要梯度的节点，那么该节点用虚线表示\n",
    "        grad_back = [v if k.requires_grad else 0 for (k, v) in node.back_prop.items()]\n",
    "        if len(grad_back) > 0 and sum(grad_back) == 0:\n",
    "            attr['style'] = 'dashed'\n",
    "        return attr \n",
    "    \n",
    "    if direction == 'forward':\n",
    "        return _forward_attr()\n",
    "    else:\n",
    "        return _backward_attr()\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _get_node_type(node):\n",
    "    '''\n",
    "    决定节点的类型，计算节点、参数以及输入数据\n",
    "    '''\n",
    "    if node.op is not None:\n",
    "        return 'computation'\n",
    "    if node.requires_grad:\n",
    "        return 'param'\n",
    "    return 'input'\n",
    "\n",
    "\n",
    "def _trace(root):\n",
    "    '''\n",
    "    遍历图中的所有点和边\n",
    "    '''\n",
    "    nodes, edges = set(), set()\n",
    "    def _build(v):\n",
    "        if v not in nodes:\n",
    "            nodes.add(v)\n",
    "            for prev in v.prevs:\n",
    "                edges.add((prev, v))\n",
    "                _build(prev)\n",
    "    _build(root)\n",
    "    return nodes, edges\n",
    "\n",
    "\n",
    "def _draw_node(graph, node, direction='forward'):\n",
    "    '''\n",
    "    画节点\n",
    "    '''\n",
    "    node_attr = _get_node_attr(node, direction)\n",
    "    uid = str(id(node)) + direction\n",
    "    graph.node(name=uid, **node_attr)\n",
    "\n",
    "\n",
    "def _draw_edge(graph, n1, n2, direction='forward'):\n",
    "    '''\n",
    "    画边\n",
    "    '''\n",
    "    uid1 = str(id(n1)) + direction\n",
    "    uid2 = str(id(n2)) + direction\n",
    "    def _draw_back_edge():\n",
    "        if n1.requires_grad and n2.requires_grad:\n",
    "            grad = n2.back_prop.get(n1, None)\n",
    "            if grad is None:\n",
    "                graph.edge(uid2, uid1, arrowhead='none', color='deepskyblue')   \n",
    "            elif grad == 0:\n",
    "                graph.edge(uid2, uid1, style='dashed', label=f'{grad:.2f}', color='deepskyblue', fontname='Menlo')\n",
    "            else:\n",
    "                graph.edge(uid2, uid1, label=f'{grad:.2f}', color='deepskyblue', fontname='Menlo')\n",
    "        else:\n",
    "            graph.edge(uid2, uid1, style='dashed', arrowhead='none', color='deepskyblue')\n",
    "\n",
    "    if direction == 'forward':\n",
    "        graph.edge(uid1, uid2)\n",
    "    elif direction == 'backward':\n",
    "        _draw_back_edge()\n",
    "    else:\n",
    "        _draw_back_edge()\n",
    "        graph.edge(uid1, uid2)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def draw_graph(root, direction='forward'):\n",
    "    '''\n",
    "    图形化展示由root为顶点的计算图\n",
    "    参数\n",
    "    ----\n",
    "    root ：Scalar，计算图的顶点\n",
    "    direction ：str，向前传播（forward）或者反向传播（backward）\n",
    "    返回\n",
    "    ----\n",
    "    re ：Digraph，计算图\n",
    "    '''\n",
    "    nodes, edges = _trace(root)\n",
    "    rankdir = 'BT' if direction == 'forward' else 'TB'\n",
    "    graph = Digraph(format='svg', graph_attr={'rankdir': rankdir})\n",
    "    for item in nodes:\n",
    "        _draw_node(graph, item, direction)\n",
    "    for n1, n2 in edges:\n",
    "        _draw_edge(graph, n1, n2, direction)\n",
    "    return graph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 用于定义线性模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mse(errors):\n",
    "    '''\n",
    "    计算均方误差\n",
    "    '''\n",
    "    n = len(errors)\n",
    "    wrt = {}\n",
    "    value = 0.0\n",
    "    requires_grad = False\n",
    "    for item in errors:\n",
    "        value += item.value ** 2 / n\n",
    "        wrt[item] = 2 / n * item.value\n",
    "        requires_grad = requires_grad or item.requires_grad\n",
    "    output = Scalar(value, errors, 'mse')\n",
    "    output.requires_grad=requires_grad\n",
    "    output.grad_wrt = wrt\n",
    "    return output\n",
    "\n",
    "\n",
    "class Linear:\n",
    "    \n",
    "    def __init__(self):\n",
    "        '''\n",
    "        定义线性回归模型的参数：a, b\n",
    "        '''\n",
    "        self.a = Scalar(0.0, label='a')\n",
    "        self.b = Scalar(0.0, label='b')\n",
    "\n",
    "    def forward(self, x):\n",
    "        '''\n",
    "        根据当前的参数估计值，得到模型的预测结果\n",
    "        '''\n",
    "        return self.a * x + self.b\n",
    "    \n",
    "    def error(self, x, y):\n",
    "        '''\n",
    "        当前数据的模型误差\n",
    "        '''\n",
    "        return y - self.forward(x)\n",
    "\n",
    "    def string(self):\n",
    "        '''\n",
    "        输出当前模型的结果\n",
    "        '''\n",
    "        return f'y = {self.a.value:.2f} * x + {self.b.value:.2f}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.linspace(100,300,200)\n",
    "x = (x - x.mean()) / x.std()\n",
    "epsilon = torch.randn(x.shape)\n",
    "y = 10 * x + 5 + epsilon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x17dca5c5d90>]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAisAAAGdCAYAAADT1TPdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABh6ElEQVR4nO3dd3xb5b0/8M/R9pDkbXnb2TtxdsJIgBKSQgqlpaxL0wFlNLdlXS5c+oNAL6QLym0p0FIaaCmFFsKmQICQANnD2dt7b0mWbcmSzu+Po3Ms2bJjO5Yl2Z/36+VXbekc+bHiJh++z/d5HkEURRFEREREEUoV7gEQERER9YdhhYiIiCIawwoRERFFNIYVIiIiimgMK0RERBTRGFaIiIgoojGsEBERUURjWCEiIqKIpgn3AM6V1+tFdXU1jEYjBEEI93CIiIhoAERRhN1uR2ZmJlSq/msnUR9WqqurkZOTE+5hEBER0RBUVFQgOzu732uiPqwYjUYA0g9rMpnCPBoiIiIaCJvNhpycHOXf8f5EfViRp35MJhPDChERUZQZSAsHG2yJiIgoojGsEBERUURjWCEiIqKIxrBCREREEY1hhYiIiCIawwoRERFFNIYVIiIiimgMK0RERBTRGFaIiIgoojGsEBERUURjWCEiIqKIxrBCREREEY1hhYiIaJRyuj34y5clOF1vD/dQzgnDChER0Sj15Mcn8eh7R/Hoe8fCPZRzwrBCREQ0Cp2ut+OFL0ukz+uGVlk5Xd+G9R8cw/Fa23AObdAYVoiIiEYZURTx8DtH4PaKAIAaWyecbs+gX2fjvkr8cWsxnvj45HAPcVAYVoiIiEaZj47U4qvTTdBpVNBrVBBFoKK5Y1Cv4fWKeLuoGgDwzcKsUAxzwBhWiIiIRpmPjtQBANYsycO41HgAQEVze5/Xd3Z54PFVYWS7S5tR1doBo16Di6ekhW6wA8CwQkRENMo0tjkBAJMtJuQlxQIAypocQa89WWdH4aObcPc/iwIef3N/FQDg6zMzYNCqQzfYAWBYISIiGmWaHS4AQFKcFrnJvrDSR2XlxW2l6Ojy4O2iauwrbwEgVVreP1QDALgqzFNAAMMKERHRqNOihBU9cn2VlfKm3mGl3eXGO76+FAB46pNTAIDNx+th73Qjw2zAooKkERhx/zThHgARERENH1EU0eQLK8lxOuT1U1l5/2AN2pxupBn1aHa4sPVkA17ZWY4/f1kMALhyThZUKmHkBt8HhhUiIqJRpN3lgdPtBQAkxekg+vpmy5vb4fWKAeHj1d0VAIA1S/NR3tSO1/ZU4H/ePAQASInX4T8W547s4PvAsEJERDSKyP0qOo0KsTo19AkGaFQCXG4v6uydyDDHAABO1dmxt6wFapWAa+Zlw+n2YuP+SnR5RKyaYcH/XjUDyfH6cP4oCoYVIiKiUaTZbwpIEARo1AKyEmNQ1tSOsqZ2Jay8c0DqVblochrSTAYAwL9uW4p2pxtLxidDEMI//SNjgy0REdEo0r0SSKc8FqzJtrhBWsq8eFx3A+2cnAQsnZASUUEFYFghIiIaVYKFle4m2+69VipbpR1tsxNjR3B0Q8OwQkREFIU6uzzY8FUJKlsCV/kEDStJcQCAMr/KSpXvvuzEmFAP9ZwxrBAREUWhJzedxCPvHsUtf90bsFV+U7BpIF9lpdy3fLmzy4PGNuk6hhUiIiIadrXWTry0rRQAcKzGhld2linPKRvCxQaZBvJVVipbpCmgeL0G5hjtSAz5nDCsEBERRagujxc/e+sQXt1VHvD47z47BafbC6NBWtT7m49PKiFFqazEB04DCQJg7ehCY5tTmTrKSoiJuGbaYBhWiIiIItSnx+rw8o5y3L/xEL463QgAKG104J++zdz+dNN8TLEYYe3owm8+PgEAaHZIhxgm+00DxejUyoGGJ2rtqFKaayN/CghgWCEiIopYnxyrVz6/558HsLesGbe9vBdur4iLJqdiyfhkPLR6GgDgjX2V8HhFtLR3AQAS/aaBAGCyxQgAOF5rV6aBshhWiIiIaKg8XhGbj0thxWTQoNbWiW89ux3Ha+1IitPhwcunAgAWFSTDoFWhs8uL8uZ2NLX5KivxPcJKuhRWTtbaUdXCygoRERGdo6KKVjQ5XDAaNHjpBwuh8Z3ps3R8Mj786QWYkCaFD7VKwETf54errLB1ugFIJy77m2wxAQCO19n9elYif48VIMRhZf369ViwYAGMRiPS0tJw1VVX4cSJEwHXiKKIdevWITMzEzExMVi+fDmOHDkSymERERFFvE+P1QEAlk1KRWFuIl6+eRGeunYO/vbDRcr2+DJ5imdHcRMAQBDQa5XPZEs8AKmyUsHKSrctW7bgxz/+MXbs2IFNmzbB7XZjxYoVcDi6d9D71a9+hSeffBJPP/00du/eDYvFgksvvRR2uz2UQyMiIopon/r6Vb42NR0AsHhcMq4qzIJa1Xv1jjzFs90XVhJjdb2uy0+Og06jQkeXBw12aaooWsJKSA8y/PDDDwO+3rBhA9LS0rB3715ceOGFEEURTz31FB588EFcffXVAICXXnoJ6enpeOWVV3DrrbeGcnhEREQRqaK5HSfq7FCrBCyfnHrW6+XKinzej/+GcDKNWoUJqfE4WmMDABi0qqDXRaIR7VmxWq0AgKQk6dCkkpIS1NbWYsWKFco1er0ey5Ytw7Zt24K+htPphM1mC/ggIiIaTTYdlaaA5uUlIiH27IFiii+syJL6uGey33XZibFRsccKMIJhRRRF3H333Tj//PMxY8YMAEBtbS0AID09PeDa9PR05bme1q9fD7PZrHzk5OSEduBEREQjSBRF/HOPtI/K5TMzBnRPqlGPhNjuHpW+Kib+YSUrITqmgIARDCtr167FwYMH8Y9//KPXcz2TnSiKfaa9Bx54AFarVfmoqKgIyXiJiIhCQRRFHK6ywu3xBn3+UJUVx2vt0GlUuGpO1oBeUxAEpW8FCNy91p//NdHSrwKMUFj5z//8T7zzzjvYvHkzsrOzlcctFgsA9Kqi1NfX96q2yPR6PUwmU8AHERFRtHjqk1O44vdf4uUdZUGff823O+3K6RaYYwd+bo9/1WSg00DRIqRhRRRFrF27Fhs3bsRnn32GgoKCgOcLCgpgsViwadMm5TGXy4UtW7Zg6dKloRwaERHRiHM43fjLVyUAgL3lrb2e73B58E5RNQDgugWDa3MICCt9TANlmA3KeULRsnstEOLVQD/+8Y/xyiuv4O2334bRaFQqKGazGTEx0uFJd955Jx5//HFMnDgREydOxOOPP47Y2FjccMMNoRwaERHRiHt9byXsvk3bypvbez3/waEa2J1u5CTFYPG45EG9tn+Tbc/da2WCIODSqen48Egt5uYmDOr1wymkYeXZZ58FACxfvjzg8Q0bNuB73/seAOC+++5DR0cH7rjjDrS0tGDRokX4+OOPYTQaQURENFp4vKJSVQGk5cn+RFHEX31TQ9fOz4EqyH4q/Zno14/S81wgf098ZzYed8+EQase1OuHU0jDiiiKZ71GEASsW7cO69atC+VQiIiIwsbp9uD9gzUoa2qH0aCBvdONZocL9s4uGA1SX8oXpxpxoKIVeo0K1y7IHfT3MBm0mJAWj9P1bchN6rsfRRCEqAoqQIjDChER0Vjm9nhx4593YmdJs/LYTYvz8NruCjQ5XChvbsf0TDNEUcTvPzsFALhxUR5Sjfq+XrJff/7ufFS1diA/JW5Yxh8peJAhERFRiOwsaVaCil6jwuxsM35wfgFyfJWP8iZpKmhHcTN2l7ZAp1Hh1mXjhvz98lPicN6ElHMfeIRhZYWIiChEPjwsLSy5Zl42fvXtWcoeYnnJsSiqaFWabOWqynULcpDe45BCYmWFiIgoJLxeER8dkcLK12dlBGx2KveUlDW3o97WiW1nmiAIwG3LxodlrJGOYYWIiCiIsiYHWttdQ75/f0UL6u1OGPUaLB0fuAxZngaqaG7HrlJpmmiqxYTMKNoCfyQxrBAREfWws7gJlzyxBT94cXfA4119bJEfjDwFdPHUNOg1gatv8uTKSlM7dvl6WhaNSzqXIY9qDCtERER+Ors8eGDjIbi9Ig5WWpWA8tyWM5j+8EfYcrKhz3tP19vx368fxD92lePfvrCycrql13W5yVJYqWrtwLYzTQCARQUMK31hWCEiIvLzh82nUdzoAAC4vSKqWzsAAJ8crYPL7cW6d470WWF59vNivLanAg9sPITKlg4YtCosm5za67p0owE6jQoer4jT9W0AgAX5DCt9YVghIiLyOV1vx7OfnwEA6DTSP5FlvuXFcoApaXTg1V3lQe+Xd6VN9p3Nc9WcLMTqei+8VakE5PidzTMhLR7J8UPbW2Us4NJlIiIin3cP1MDtFXHhpFTo1AI+OVbva7Q1o9nR3Wz71Cen8M252YjXB/4zWtkihZU/fXc+cpJi+t32Pi85DmcapADEKaD+MawQERH5yPueLBmXjMY2JwCpslLiq6qkGvUw6jUobnTg8t99gdnZCfjm3CxcNDkNXR4vam2dAICcxBikGfvfL8V/S/yFDCv94jQQERGRT1mTFEpyk2KRl9y9F0qxrwIyITUe674xHXqNCmVN7XjnQDV+8sp+eL0iaq2d8IqATq1CygCmdHIYVgaMYYWIiMinvFlqppXCinS+TlmTQ6msFKTG4cJJqfjq/oux4fsLoFULsDvdqLF1osrXiJuZYBjQicnjUqXXz0uORYaZ+6v0h9NAREREABxOtzL1k5scC6NB+ieyvLkdZxqkFTvjfAcEpsTrcdHkNOQnx+FUfRvO1Leh3i7dm53Y94nH/pZNTMW9KyZh0bjks188xjGsEBERAajwNceaY7Qwx2gRq1NDrRLQ2eVVNm6TqyGy8anxUlhpaIOtww0AyBrgLrQqlYC1F08cxp9g9OI0EBEREbpPQJZ7VbRqlRI8mnwrgQpS4gPuGZ8mhZczDW2oapXuz0rklM5wY1ghIiJC90og/8ZXObgAgEYlILtHEBmfKoWX0/VtqGyRelYGWlmhgWNYISIiQndYyesjrOQmx0KrDvxnUw4rZxocSoNtz0BD5449K0RENGZtPl4Pt1fEpdPSlZ1q/fc/yU/u7lGRm2v9jU+TwkqD3Qm1bwUQp4GGH8MKERGNOZ1dHjzy7hH8Y1cFBAHYcu9Fylb5/mHF//OCIGElXq+BxWRAra0THq8ItUqAxdT/ZnA0eAwrREQ0JhyrseF/3z+KdpcH9TanMm0jisDbRVVKz0mu39RPvl9AGZca2FwrG58Wp+xcazEZoFGzw2K48R0lIqIx4S9fluCr003YX96KqtYOJMXpcO38HADAX3eUweXxQqsWAjZoO1tlBejuWwE4BRQqrKwQEdGoVNHcDkHo3qRtb1kLAOCeSydhaoYJ8/ISIQjAxv2VaPDb0E3tt/usQavGwvwknGlow7RMU9Dv4x9WsrkSKCQYVoiIaNTp7PLgG09/CZUg4Iv/vgjtLg+KfVvm37QkDwl+pyFfMDEVnx2vBxC4bFn2jx8thsvtRYxOHfR7TUhjZSXUGFaIiGjUqWzpQEt7FwBg68lGCL5iyaT0+ICgAgCrZ2coYSUvSFhRq4Q+gwrQYxqIlZWQYFghIqJRp8baoXy+6WgdkuK0AIB5eb1PN/7a1HToNCq43N6AHpWBSjfpEadTw+HyDPhcIBocNtgSEdGoU2PtVD7/7HidcrbPgvzEXtcaDVp8e142VAKwZPzgDxUUBAE3LcnHrGwzCnMThjxm6hsrK0REFBVO1tlx45934o7l4/H98wr6vbbWL6y0tHehpd0KAJgfpLICAI9+Yzr+a8VkJMbpgj5/NvevmjKk+2hgWFkhIqKo8E5RNRrsTrzwZQlEUez3WnkaSOhe2IM0ox45ScF7SjRq1ZCDCoUewwoREUWFA5WtAKTm2TMNbf1eK08DXTQ5TXlsfn4iBP/0QlGDYYWIiCKeKIo4VGVVvt58vKHf6+VpoO/Mz4FeI/1T19cUEEU+hhUiIop4Fc0daPUtRQaAz0/W93t9tW8r/fGpcbhxUR5SjXpcNsMS0jFS6DCsEBFRxJOngFKNegDArpJmtDndQa91ON2wdUrPWcwGPLR6GnY/+DXugRLFGFaIiCjiyVNAl01PR35yLLo8Ir463Rj0WrlfxajXwGjQjtgYKXRCGla2bt2K1atXIzMzE4Ig4K233gp4/nvf+x4EQQj4WLx4cSiHREREUehARSsAYFZWApb7mmY/O1YfdFWQ3K9iMRtGbHwUWiHdZ8XhcGD27Nn4/ve/j29961tBr1m5ciU2bNigfK3TcekYERF183pFHPZVVmblmJFm0uPFbaV4bU8FPjhUg3Fp8ZiRaUJhbiJWz85Qli0zrIweIQ0rq1atwqpVq/q9Rq/Xw2Jh0xMREQVX3NgGh8uDGK0aE1LjUZASh4UFSdhd2gy7040DFa04UNGKv+8sR4PdiS6PFwCQwbAyaoR9B9vPP/8caWlpSEhIwLJly/DYY48hLS2tz+udTiecTqfytc1mG4lhEhHRMCiqaEWm2YA008CDxIEKqaoyPdMEjVoFDYB/3roEnV0elDe340StHR8eqcX7B2vw4ZFaTMswAQAyzGyoHS3C2mC7atUq/P3vf8dnn32GJ554Art378bFF18cEEZ6Wr9+Pcxms/KRk5MzgiMmIqKh2lnchKv+8BV++mqR8liLw4VtZxr73ZH2qzNSI+2s7ISAxw1aNSalG7F6diYevmIaAKm35VBVKwBWVkaTsIaVa6+9FpdffjlmzJiB1atX49///jdOnjyJ999/v897HnjgAVitVuWjoqJiBEdMRERD9cqucgDSGT+yR987ihue34l/7gn+d/nGfZXYuK8KAHDJ1L6r7mkmA2ZnmwEAh6ukijt7VkaPiFq6nJGRgby8PJw6darPa/R6PUwmU8AHERFFNmtHFz48XAsAaG53we3rKzlRKwWXZz4/A483sLqyt6wZ979xCACw9qIJOG9CSr/f45Kp6QFfZ3JflVEjosJKU1MTKioqkJGREe6hEBHRMHr3QDWcbimgiCLQ7HABAOrt0rR/WVM7/n24RrleFEXc/c8DcHm8uGx6Ou6+dNJZv8fXeoQVVlZGj5CGlba2NhQVFaGoqAgAUFJSgqKiIpSXl6OtrQ333nsvtm/fjtLSUnz++edYvXo1UlJS8M1vfjOUwyIiohH2rx7TPPV2J9weL5oc3T2Kz205o/SuVLZ0oKypHVq1gN9cMxsq1dkPIJyaYUSmL6DE6dQw6sO+hoSGSUjDyp49e1BYWIjCwkIAwN13343CwkI89NBDUKvVOHToEK688kpMmjQJa9aswaRJk7B9+3YYjcZQDouIiEbQiVo7DlRaoVEJSphoaHOi2eGCKAIqAYjRqnG4yoYvfbvS7ilrBgBMzzQPeBdaQRCUqaCMhBiesDyKhDR2Ll++vN8O748++iiU356IiCLAG/sqAQAXT0lDp9uLamsnGu1O1MdLVZWUeD0un5WBDV+V4m/by3DBxFTsLm0BACzITxzU9/rm3Cz8Y1f5oO+jyBZRPStERDS6eL0i3jtQDQC4em4WUuOlgwgb2pyot0vb4qca9bhmnrQNxZaTDXA43djrCyvz8pIG9f3m5ibiq/svxrpvTB+uH4EiAMMKERGFzP6KFlRbOxGv12D55DTl1OQGuxMNvubaNKMeUzOMyEuOhdPtxdtF1TjhW948fwgVknSTAXqNevh+CAo7hhUiIgqZdw9IK3xWTEuHQatGSrx0/ltjmwv1NjmsGCAIAlbOkI5e+e0nJwEA41LikOKrxNDYxrBCREQh4fGKeP+QFFZWz84EAL/KSqeybFl+bOV0i+856fF5eew7IQnDChERhcTOkiY02J1IiNUqG7oFnQYySY/Nzk4I2CJ/Qf7g+lVo9GJYISKikJCngFZOt0Cnkf65SfOFlcY2l9JgKz+mUgm4zFddAYB5XNFDPgwrREQ07DpcHrx3UFoFJE8BAVB6UKwdXahs6QDQXW0BgCtmSTuYpxn1GJcSN1LDpQjH7f2IiOictLvciNUF/nPy78M1sHe6kZMUgyXjkpXHzTFaaNUCujyi0rOSZuye+pmfn4Q/3DAX2Ync1I26sbJCRERD9o9d5Zj+8Ef45+7A7fRf3SV9fe38nICt8gVBUPZakflXVgDg8lkZmJ2TEJoBU1RiWCEioiH74lQDRBF47INjaPEdTnimoQ27SpuhEoBv+zZ785fiF06MBg0MWu6JQv1jWCEioiGraJb6TqwdXXjKtz/Ka74qy8VT0oKefOxfWUkzch8VOjv2rBAR0ZBVtLQrn7+8sxxdXhFv7a8CAFy7IDfoPf7TPv79KkR9YWWFiIiGxN7Zhdb2LgDA+RNS4PGKeGVnOdpdHhTmJuCiyalB7/PflbZnvwpRMAwrREQU1N92lGHGwx9hR3FT0OflpccJsVr84lszUZibgJXTLXhhzXz869Yl0KiD/xMTWFlhWKGz4zQQEREF9fqeCrQ53fjD5tNY7Lf8WCaHlZzEWGQnxuLNO84b0OsGhBUTwwqdHSsrRETUS4fLgyPVNgDAF6caUdrogMcr4jcfncDfd5YBACqapX6V7MSYQb02p4FosFhZISKiXg5UtsLtFZWv/7G7HAkxOjy9+TTUKgFXzcnqrqwkxQ7qtdlgS4PFsEJENMq1Od34wYbduHhqGm5bNn5A9+wtawEApMTr0Njmwj92lqOjywNAOk35YKVVWQmUM8jKCntWaLA4DURENMrtKmnCrtJm/HVb6YDv2ecLKz+6cBzSTXrYOt3o8nRXWvZXtCiVlezEwVVW4nRqTEyLR0q8btBVGRqbGFaIiEa56lbpdON6uxNev6mdvoiiiL3lUlhZWJCMa+dLu9Amx+lw67JxAIB9Za2o9PWs5CQNrrIiCALe/c/zsfne5dy9lgaE00BERKNcdatUAXF7RTQ5XGdtaj3T4EBrexf0GhWmZZgwIS0eHV0eXD4rE26PF3/cUowdxU1oc7oBDL6yAgAGrZpBhQaMYYWIaJSTwwoA1Nk6zxpW5Cmg2TkJ0GlU0GlUePDyaQCAzi4PNCpBCSop8XqGDgo5TgMREY1y8jQQIIUVWY21A2/tr8IvPzyOkkaH8rjcXDsvL7HXaxm0akzLNClfD3YKiGgoWFkhIhrlqgIqK04AwB82n8avPzqhPL79TBPevGMpRBHYUSLtWDsvt3dYAYDCnAQcrLQCGNoUENFgsbJCRDSKebwiav2qKfLnHx2pBQBMTjdCr1GhqKIVX55uxFtFVShraodRr8HCcUlBX7PQL8QMdtky0VAwrBARjWINdic8fiuA6m2dEEVRmfb53fWFuGGRdDrybzedxBMfnwQA3H7ReJgM2qCvWZiboHzOpcc0EhhWiIhGAbfHi28/uw3feW57QDjxnwICpMpKa3sX7J1Sg2xuUixuvXA8dGoV9pW3oqq1AxlmA35wXkGf3ys3KRZJcToA0rlARKHGsEJENApsO9OEPWUt2FXajLKm7mZZeSWQIEhf19mcKPU9bzEZEKNTw2I24DsLspV77r50Ur8rfARBwM+vnIHvn5ePJeN7H3BINNwYVoiIRoF3DlQrn5+ssyufy2FlUpoRgDQNVNYkbeaWl9xdFbl9+QQkxmoxLy8RV8/tDi59uXxWBh5ePR1qlTAs4yfqD1cDERFFOafbg48O1ypfH6+1Y+WMDADdYaUwNwEn6uxocriUMJOfHKfck5UQg+0PXAK1SmAAoYjDygoRUZTbcqIBdt8mbUCPyopVWv0zPdMEnVr6K39PqbSPSl5KYL+JQauGVs1/Fijy8LeSiChCNLU5ccovaPTnk6N1OO8Xn+H/PjmFjfuqAABTLNJUz/Ha3tNAWYkxSDNJO9cWVbQCCKysEEUyhhUiogjx3b/swsr/+wI11o6zXvvm/ipUtXbgt5+cxIe+PVPuWTEZAFDa6EBnlwdAd1jJTIhBuskAAHB5vAACe1aIIhnDChFRBOjyeHGsxgaPVwyojPTlWK0NAJSpnbzkWHxtahrMMVp4ReB0fRvaXW60tHcBADLMMbD4woosj5UVihIhDStbt27F6tWrkZmZCUEQ8NZbbwU8L4oi1q1bh8zMTMTExGD58uU4cuRIKIdERBSRaq2dkLdHqWxu7/fazi4PSn2bur299jz812WT8Ycb5kIQBEz2TQWdrLMrZwLF6zUwGTTKNBAgHUAYr+caC4oOIQ0rDocDs2fPxtNPPx30+V/96ld48skn8fTTT2P37t2wWCy49NJLYbcPbM6WiGi0qGjpDiiVLf1PA52qa4NXBBJjtZhiMeLHF03AjCwzAGn7fAA4UWdXppMyEwwQBCGgspLPKSCKIiGN1atWrcKqVauCPieKIp566ik8+OCDuPrqqwEAL730EtLT0/HKK6/g1ltvDeXQiIgiin9AOVtYkaeAplhMEITAZcZyZeVErR3jUqRpnswE6fyedL+wwikgiiZh61kpKSlBbW0tVqxYoTym1+uxbNkybNu2rc/7nE4nbDZbwAcRUbTzDyj+VZZgTvh6WqZkGHs9J4eVo9U2/GtPJQAgO7F3WGFlhaJJ2MJKba3UvZ6enh7weHp6uvJcMOvXr4fZbFY+cnJyQjpOIqKRUDmIaaDjvsrKVIup13PKTrV2J/aUtcCo1+C7S/IBAOl+PSv5KaysUPQI+2qgniVMURR7PebvgQcegNVqVT4qKipCPUQiopDzDyjNDhccfpu89SRXVuQqij9zrBYZZqmCoteo8ML3FmCSr48lsLLCsELRI2xhxWKxAECvKkp9fX2vaos/vV4Pk8kU8EFEFO2qelRTep6WLGuwO9HY5oIgQAkhPa2cYUGcTo1nbpyLhQVJyuNxeg1mZpmRZtRjQlr88A2eKMTCtm6toKAAFosFmzZtQmFhIQDA5XJhy5Yt+OUvfxmuYRERDYooivjuX3bB5fbilVsWD+lcnS6PV1m5k2rUo8HuREVzuxJGRFHEk5tOIkanxkzfqp+C5DjE6IKfjPzw6ul4YNVU6DS9/3t04x1L4fGK/Z6qTBRpQhpW2tracPr0aeXrkpISFBUVISkpCbm5ubjzzjvx+OOPY+LEiZg4cSIef/xxxMbG4oYbbgjlsIiIhk1Lexe+ONUIAKhobh9SL4i8x4pOo0JhTgI+PloXMC20p6wFv/9M+rtUXuETrLnWX7CgAgBatQrMKRRtQhpW9uzZg4suukj5+u677wYArFmzBi+++CLuu+8+dHR04I477kBLSwsWLVqEjz/+GEZj//8nJCKKFPX2TuXz4sa2IYUVefVPdkIMcpOkVTr+Dbdv7q/y+x7SZnCT0zkFTmNHSMPK8uXLIYpin88LgoB169Zh3bp1oRwGEVHINNidyudn6h24eMrgX0OuomQlxijLjOXHnG4P3j9YAwC4bHo6PjpSByB4cy3RaMW9lomIzkG9rTusFDe2Dek15GCSnRiL7ESpsiJXWz4/0QBrRxfSTXo8c+M8PP9FMfaWtWDZpNRzHDlR9GBYISI6B/X+lZUGx5BeQ57yyU6MQXZSYGXlzX3SFNCVc7KgVgm4bdn4cxkuUVQK+z4rRETRLKBnpWFwlZXWdhdEUfSrrMQolZXW9i5UtXbgs+P1AICr5mQN04iJog/DChHROfCvrDS2uWBt71K+djjd+OYzX+FXHx7vdd9nx+sw59FNuOWve5QTlLMTYxGv1yAxVgsAuOWlPXB5vJicbsS0TDbU0tjFsEJEdA4a/HpWAOCMX9/K3rIW7C9vxR+3Fgc04gLA33eUAwA+OVavBJ4cX3OtXF05WmODQavCw6unhWz8RNGAYYWI6BzI00AGrfTXabFf34ocUDxeEW8XdS8/trZ3YeupBgCA0SC1Duo0KqTES2f35Pj6VowGDV7+4SIsnZAS4p+CKLIxrBARnQO5KjI/T9rW3r9vpaGtu5qycV93WPnoaC26PCImpxvxztrzMTc3Af+xKA8q3+63N18wDlfMysBrP1qC+fnd2+UTjVVcDURENERtTjfaXR4AwJLxyfjydCPO+IcVv6mfozU2HKuxYWqGCe/59k25YlYGClLisPGO8wJed25uIubekDgCPwFRdGBlhYjGPFEU0dnlGfR9chiJ06kx3dcA6z8N1OirrMjHBW3cV4lmhwtfnZa2579idua5DJtozGBYIaIx74cv7cGS9Z/idP3glh7X26R+lTSTAeNTpVOMy5ra4fZ4AXSHmVUzMwAAr+6uwO0v74XHK2J6pgkFQ9ian2gsYlghojGtzenG5hP1aGnvwv9sPASvt+8jQnqS+1VS4/XISoiBXqOCy+NV9k2Rw8q352XDYjLA3unGzpJmAMAVs1hVIRoo9qwQ0Zh2qNIK+QizXaXNeG1PBa5fmAtAmh76r9cP4mBlK2J1GoxLjcP6q2dCr5GOLVbCikkPlUpAQUocjtfalQMN5QbbrIQYvL32POwsaUZZowMujxffW5o/4j8rUbRiWCGiMe1AZSsAwKjXwO504/EPjuGSKWlIMxlQ0ujA63srlWuLKlqxelYmLpqSBqB72XKaUVpynJ8shZXSxnY43R60+jaIS43XIzFOh2+wR4VoSDgNRERjWlF5KwDg9ovGY2aWGfZON17fJwWU0iapWbYgJQ4LfUuIy5r89lHxbQiXZjQAAPKSpc3cypvb0dTmAgBo1QLMMdrQ/yBEoxjDChGNaXJlZV5uIlbOsAAATtVJjbbyyp6pGUbMyU0AAFT4+lGA7mkgubKS6xdW5H6VlHi9sn8KEQ0Np4GIaMyqs3WixtoJlQDMyDKjxTdtI68Kkisr+clxyDBL1ZPy5nblfmUayCSFlbwkaXVPWZNDCSupviBDREPHsEJEY1ZRRSsAYFK6EXF6DSakScuPzzS0wesVUdLYPQ0kh44Kv7DSYA8+DVTR0oE6X5BJjWdYITpXDCtENOa0u9yI1WmUsDInJwGAFDY0KgHtLg9qbZ0obZSCSUFKHBLjdACksCKKIro8olKJkaeBMswGaFQCXG4vDldZAUA574eIho5hhYjGlI+P1OKOv+/DwoIk2DvdALrDilatQl5yLM40OHCk2oaqVqk/pSAlDnF6DQQBcLg8aHa40On2+u4RkBArNdBq1CpkJ8agtKkde0pbAHAaiGg4sMGWiMYMp9uDR987CrdXxLYzTTjkq37IzbMAlKmgT4/VAZBOPk6K08GgVcNikqZ7Klo6lN1rU+P1EITuBtrcZKlv5ZSv74VhhejcMawQ0ahWb+tUGmZf3lGOypYOpBn1ylk+8XoNJqYZlevlsPLJsXoAUlVFDiM5id2rfcqapCkii6/xVpaXFBvwNcMK0bnjNBARjVqiKOJbz21DRXMHrpqTiS0nGwAAd106CVfPzcJL20oxPjUear+lxXJYkQ8h9D+/JycpFrtKm1HR3I7KFimszM0NPB1ZbrKVMawQnTuGFSIatSpbOlDRLPWdvFVUDQAYnxqHa+ZlQ6NW4UcXju91z4RUY8DX+cn+YSUGgNRku8t3xs+icckB1+f2rKywwZbonHEaiIgiSofLA1Ec+GGC/ZF7UnKTYjE3NwE6tQoPr54Ojbrvv/rGpcb1+bUcRPaWtaC40QFBgLKzrSwvOfB+VlaIzh0rK0QUMY5W23DVH77Cd5fk4WdXTDvn15PDynkTUrD+6plwuj3KIYR9idNrkGk2oNoqNdAGVlaksCI3z061mGCODdxK37+yEqtTI07Pv2aJzhUrK0QUMbaeaoDL48Vnx+uHdP/+8has+O0WfHZcWslzqFIKKzOzzABw1qAiG+/rWwGA/JTelRXZonGBVRUAiNGplX1XWFUhGh4MK0QUMU7W2gFI29w73Z5B3SuKIn7+3lGcrGvD7z49DVEUlcrKrGzzoF5LbrJNjtMFHEKYGq+HXtP91+aiguRe9wLdTbbsVyEaHgwrRBQ2bxdV4ZF3j8DjlXpUjvvCileEstX9QO0ubcE+3wnKRRWt2FHcDGtHF3RqFSalG/u/uQc5rPivBAIAlUpAdmKM8vXCgt6VFQDI9Z0RxN1riYYHwwoRhc3jHxzDhq9K8dXpRrg9XpxuaFOeO1nX1s+dvT37+emAr3/90XEAwJQMI3Sawf1Vd8WsTFxdmIWffm1ir+fkqaApFiOSfFvw9zTNt4dLz2ZdIhoadn4RUVi4PV7U+w4C3FPajKzEGLh8W9gDwOk6e9D7XG4vNnxVggUFScoeJ8dqbNh8ogEqAVizNB8bvipVqixyv8pgmGO0ePLaOUGfG5caj80nGrB4XPApIAC4cVEushNjcN6ElEF/byLqjZUVIho2oiji7zvLsKe0+azXNra5IK9Q3lXajBO1geFEXnHT03NbzmD9v4/j3n8dUB57/otiAMDXZ2bgtmWBe6cMJaz059Zl4/DTSybiziBVF5lBq8Zl0y2I50ogomHBsEJEw+ZEnR0PvnkY971x8KzX1vnO1gGkHhP5lOIM3/b1wcJKRXM7/rBZmu4pbnCgurUDHq+IT31b469Zmo90kwHz87p3lZ05yObas0kzGnDXpZOQEBt8CoiIhh/DChENm1rf3iT1NudZr/UPK51dXrzt22H28pkZAIDSRkfAtBAAPPLuETj9Htt2pgkHKlth7eiCyaBBoe/05FW+19BpBt9cS0SRh2GFiIaNtaMLANDmdMPt8fZ7bZ09MNBUtUrb4i+bnIp4vQZur4jSpu4VQR8cqsEnx+qhUQlYOd0CANh2uhFfnGwEIG38Ju9Me+WcTIxLkbbV1/azWy0RRYew/7943bp1EAQh4MNisYR7WEQ0BHJYAQBbp7vfa+t9lRWtWgh4fLLFqCwdPuVbEbTtTCPufK0IAHDzBeNw05I8AMBXZxqx9ZR0OOGFk1KV10iJ1+Oze5fjsW/OPIefhogiRUR0f02fPh2ffPKJ8rVaPbBdJokosrS2d4cVa0dXn0t7ge5poAsmpio71ibF6ZAar8fEtHgUVbTiVL0d+8oNuPmlPXC5vbh0WjruWTEJHq8InUaFOpsTdb4pJ/+wQkSjS0SEFY1Gw2oK0SjQM6z0p9YXMi6ZmoavTjfC6fZiUno8BEHAxHSpsvLvQ7V4fmsx2l0enD8hBb+/vhBatQpaNTAvNxHbi5sASCcpZyXE9Pm9iCi6hX0aCABOnTqFzMxMFBQU4LrrrkNxcXG4h0REQ9Da4er+vN3Vz5Xd00A5ibGY7WuMnWKRNlOb6GuKPVFnh8PlwdLxyfjTd+fBoO2uup43oXufE1ZViEa3sIeVRYsW4a9//Ss++ugjPP/886itrcXSpUvR1NQU9Hqn0wmbzRbwQUSRwdYx8MqKPA2UbjLgxkW5SIzVYvVsaRWP/wqer8+0YMP3FyBWF1gIXuq34RrDCtHoFvZpoFWrVimfz5w5E0uWLMH48ePx0ksv4e677+51/fr16/HII4+M5BCJaID8p4Fs/YQVp9uDFt+16SY9JluycOWcLOX5rIQY3HPpJAgCcPvyCVCrhF6vMSvLjHEpcXC6vVjcx4GCRDQ6hD2s9BQXF4eZM2fi1KlTQZ9/4IEHAkKMzWZDTk7OSA2PiPrROsDKirwPi06jCjjV2N9/XtL3DrEAoFGr8N5PzodXBGJ0bMonGs0iLqw4nU4cO3YMF1xwQdDn9Xo99HqeZEoUiawDDSt2eQpID0HoXTUZqJ5TQ0Q0OoW9Z+Xee+/Fli1bUFJSgp07d+Lb3/42bDYb1qxZE+6hEdEgiKII6wBXA8nLjdONhpCPi4iiX9j/s6SyshLXX389GhsbkZqaisWLF2PHjh3Iy8sL99CIaBA6ujxw+e1a2zOslDU5sPaV/fjRhePQ2OYLKyaGFSI6u7CHlVdffTXcQyAa89qcbqx75whWz87EsiGurPFvrgV6h5XX91biUJUVv/roOL7uO7snzcQpXSI6u7CHFSIKv83H6/H63kqUN7UPOaz0DCfWjsDt9veXtwIAKpo78PGROgCsrBDRwIS9Z4WIwq/JNy3TfJaN3PrTs7Liv3TZ6xVxoKJV+bqkUTqg0MKwQkQDwLBCRMqS456BYzCsvt1rk33nAflXWs40tMHu7H2wIaeBiGggGFaISAkpto4uiKJ4Tq+RkxQLQOqDcfsabvf7qirz8xKVMANwGoiIBoZhhYjQ4pv+cXm86OjyDPi+t/ZX4Tt/3I56e6dSScn1hRUAsHVK1RS5X2VefiKumJWhPM+wQkQDwbBCRMrW98DgpoJe+LIEu0qa8dGROmUqKTleh3i91LsvB5giX2WlMCcRVxVK2+onxXVfR0TUH/5NQUSw+jXWWju6kJkQc9Z7RFFEcUMbAOBMfRucbmnKJyFGB5NBgzanG9aOLjicbpyolQ4cLcxNQLrJgP+7bg5SjexXIaKBYVghoiFVVupsTjhc0pTRqXo7EmKkXhRzjAamGC2qrdLU0KEqK7wikGk2KNM+/ocWEhGdDcMKESk9K0D3qp6zkasqAHC6vg0T0uIBAAmxOuVwQmtHF6paOgAAc3IThmm0RDTWsGeFaIxze7ywd3YvK5b7TF7aVorv/HE7WhzBw8sZ314pgFRlqWiWQok5VhsQVooqWgAAc3ISQjF8IhoDGFaIxrieO8/K00AvbS/FrpJmvHeoJuh9/pUVAChvbgcAJMR0hxVbRxcOVloBALOzE4Zz2EQ0hjCsEI1xLT16VORVPQ2+k5G3n2kMet+ZBkfQx81+YeVMfRtqrJ0QBGBGlnm4hkxEYwzDCtEoV9XagRue34FNR+uCPt/aY4t9a0cX2l1uZcfZHcXN8Hp7bxQnV1amZZgCHvfvWfnitBR0JqTGI47LlIloiBhWiEa5N/ZWYtuZJry0rTTo871OS27vQr2vqgIAzQ4XTtTZA67p7PKgqlXqUblsuiXgOZNBA3OsFFYa7NLrzOIUEBGdA4YVolFuf7nU4Fpt7Qj6fEuPykprhwv1dmfAY9vPNAV8XdrkgCgCRoMGi8YlKY8bDRpo1CqlsiKblc0pICIaOoYVolFMFEXlXJ6a1k7l3J/1HxzDqv/7AvbOLqWyYjJ07zpbb+8MeJ1tvrDicLrh9Yoo9vWrjE+Nx0TfkmUASkgxMawQ0TBiWCGKQqWNDvxteym6fAcF9nldU7sSRjq6PLB1SH0or+6uwLEaG/aUtiiVlYKUOADStJA8DSSf87OzpAlv7K1E4c83Yc2GXTjpmxYalxqH5Hg9En3TPgm+//WvrGhUAqb26GshIhoMhhWiKPS/7x/D/3v7CD4+ErxpViZPAcmqrR2wtncpy5WLGx3K6p+8ZCmsWNu7lGmgiyanwmjQwN7pxj3/OgCX24svTjXi+a3FAKTKCoDuDeGUXWy7w8pkixEGrfqcfl4iGtsYVoiikNzcWtLY1u91+3qElRprBypa2pWvixvalNVA+clSFcXudKPG199iMcdgUUF3T8r5E1IAQNlmf5yvGjMhzQgASmOtf1hhcy0RnSuGFaIo1OyQKh9VrZ39Xre/vBUAoNNI/1evbu1ERXN3WClpdKDFEVhZAYBTdVIISjPqcc38HBj1Gty7YhL+9sOFuGZetnLdOF9lZbGvyXZyui+0BIQV9qsQ0bnhxgdEUUYURTT7tsCvbg2+wgcA2l1uHK+VeksunJiKT47VocbaAYeze2v9kkYHEmKlqZsUox7xeum05DO+PVTSTQacPzEFK9alQxAEAMC6b0zH6YY2dHm8GJcqBZxvzM7E7OwEpcdFq1bBZNDA1unmzrVEdM4YVoiijN3pRpdHWtVT1U9YOVRphccrwmIyYG5eghRWWjvRquveV6XG2omOLmlKR94mv83phtMtNe6mmfQAoAQVAIjTa7Dx9qUBjwmCgPyU7soMADx+9UxUtnRgaobxHH9iIhrrGFaIokxTW/e+KNWtHRBFMSA4yOQly4W5Ccg0x0jXWzugVQfO/sqrhRJjdUiI1QYEoDSjPugYgn2/nq6YlXnWa4iIBoJhhSjKyP0qANDu8sDa0aVM5cgOVVrxxy1nAADz8hKRYTYAkCopcszQqgWlQgMACXHagF4Tnab35m5EROHABluiMDtRa4e9s+vsF/r4V1aA3lNBu0qacf3zO9DS3oXZ2WZcuyAHmQlSZaXG2qlcvyC/e5WPWiXAqNco+6QAQGq8fkAVFCKiUGNYIQqjfeUtuOyprfivfx0c8D1yc62s2m9FkCiKuOu1IrQ53Vg8Lgl/v2UxjAYt0k0GCALgcnvR5RGhVQtYMi5ZuS8hRgtBEGCO6a7QpJuCTwEREY00hhWiMNpRLG1jv/lEPZxuz4DuaeoRVqr89k1pcrhQ1doBQQCe/+58xPtOOtZpVEiJ7w4fWQkxykZuQPCdZ9OMhkH+NEREocGwQhRG8n4mTrcXByqsA7qnV2XF2l1ZkbfBz0mMhdEQ2G+Sae4OHzlJsShI7V69k+jrefGfBkpjZYWIIgTDClEYyeEC6H2ycV/ksCKv1PHvWZHDz6T0+F73ZfhWBAHSmT/5fpvAySElIaCywrBCRJGBYYUoTDxeEafru7fLl6eEzkaeBpJ3hvXfGE4OPxPTe+9tkpHQXVnJTYqFQatGlq/xVl5NxGkgIopEDCtEYVLe3A6n2wt5wc2+8hZ0dp29b0VeujwjSworVS1+lRVf+JmY1ruyktmjsgJ0n7Qsn5ps5jQQEUUghhWiMJGrINMyTEiJ1/v6VlrPel+zb+nyTF9Yqbc74XR7IIoiTvlec9JZKis5vrAiV2fk3WcT/FYDsbJCRJGCYYUoTE76zu2ZnG5UDgLcUdzc7z2iKCrTQBPTjND7DiisszrR2OZCS3sXBAEYn9p/z4ocVv7z4ol45eZF+M78HACsrBBRZGJYIQqTk74pm0kWI5aMl/Y8+eJUQ8BBgz21uzzKuT3J8Tql56SqtQOn6qXwk5sUixidute941PjoNeokJ8cq/SmxOjUWDohRdmCPzVej3STHjlJMUjqsSsuEVG4RERYeeaZZ1BQUACDwYB58+bhiy++CPeQiEJOrqxMSo/HYt8GbXvKWjD94Y9w+e++QGObs9c98kogvUaFWJ1a2Zm2urVDWQk0MS34wYEJsTp8dOeF+NdtS/sck06jwqa7l+HDn14IlYq71xJRZAh7WHnttddw55134sEHH8T+/ftxwQUXYNWqVSgvLw/30IhCpsvjRXFjd7gYlxKHmxbnKRu3Ham2YfPxegDSqqEnPj6BL041KFNAyXE6CIKATF8fSlVrh99KoN5TQLL8lDiknmVJssmgRZyex4YRUeQIe1h58skn8cMf/hA333wzpk6diqeeego5OTl49tlnwz00opApa3KgyyMiTictHxYEAT+/agb2/Oxr+O6SPADdDbibj9fj95+dxr3/OqCsBEqKl6ZoshKk3pNPj9XhcLUNQPA9VoiIollYw4rL5cLevXuxYsWKgMdXrFiBbdu2hWlURKF3olaqqkxIN/aabpmaYQIAHPdNEx2qkna2rbM5saukBQCQFCdVR66ckwmjQYMDlVZlJVFf00BERNEqrGGlsbERHo8H6enpAY+np6ejtrY26D1OpxM2my3ggyjayFWTyUGqIJMtUtg44QsrR6q7f8ffPVANQJoGAqRpnee/Ox8636oglYCAM3+IiEaDsE8DAeh1DL0oin0eTb9+/XqYzWblIycnZySGSDRgBypacd/rB9Bg790gKztY2QoAmGIx9XpO3iOl3u5Ei8OFI9XdZwbJW+snxXWv1Fk8Lhm/u64QapWAWdkJMGh7rwQiIopmYQ0rKSkpUKvVvaoo9fX1vaotsgceeABWq1X5qKioGImhEg3YH7eewT/3VOLVXcGbxL1eEXvLpOmc+fmJvZ6P12uQnSit8tle3IQav4MKZf5hBQBWzrDg83uX468/XHiuwyciijhhDSs6nQ7z5s3Dpk2bAh7ftGkTli4NvrxSr9fDZDIFfBBFkkrf9veHq4Ofony6oQ22TjditGqlP6WnKb6poI37KgEAOUkxMPqt0EmO670HSk5SLEw9TlomIhoNwj4NdPfdd+PPf/4z/vKXv+DYsWO46667UF5ejttuuy3cQyPqU7vLjb/vLEO9rXfVo7pVeuxwVfB+qt2l0i61c3ISlM3YepL7Vj4/0QAAmJWdgEW+vViA3pUVIqLRLOybKVx77bVoamrCo48+ipqaGsyYMQMffPAB8vLywj00oj5t3FeFn711GIerbFh/9Uzlcafbo2zmVtXagRaHC4k9gsXeUmkKaEGQKSDZZF8vi9srAgBmZJqh16jwybE6AEByPLfCJ6KxI+xhBQDuuOMO3HHHHeEeBtGAVfsaXStb2gMer+3RX3K0xobzJqQEPLbH168yLz+pz9eXp4Fk0zNNSDd1HywYbBqIiGi0Cvs0EFE0amnvAgA0+U5AlsmrdWSHqwL7VuptnShvbocgAIW5CX2+fkFKHLTq7hVx0zNNmJQej1nZZuQlxyrb7BMRjQURUVkhijat7VJIaXIELk+uaQ2srByutkEURby8sxwalYB4X5Ps5HRjv82wWrUK41PjcbzWjkyzQZn2eeuO8+ARxT57XYiIRiOGFaIhaJHDSpsrYF8geXoow2xAjbUTR6qs+PxkA/7fW4cBAAatFDKCLVnuabLFiOO1dkzLNCuPqVQCVOABg0Q0tvA/z4iGoNU3DeT2irB1uJXHq61SWLl0mrRPUHGjA7/893Hl+c4uLwBgfl7f/Sqyy2dmAABWz84YnkETEUUpVlaIhkAOKwDQ0OaEOVaa0pGXLc/INCPTbEC1tRPHa+0waFV4/baleObz0yhpbMdFk9PO+j1WTLegZP3X+9zNmYhorGBYIRoCeRoIAJranMp5PPI0UGZCDKZnmVHtWx100+I8zMgy45kb5w3q+zCoEBFxGoho0DpcHjjdXuXrJocUXERR7O5ZSTBghq/XxKBV4UcXjh/5gRIRjRIMK0SD5F9VAaTKCgDYOt1wuDwAgExzDK6YnYGshBjcd9kUpBq5iRsR0VBxGohokPz7VQCg0bfXSo2vuTYxVosYnRrjU+Px1f0Xj/j4iIhGG1ZWiHxEUURnl+es17X2rKz49lrx71chIqLhw7BCUW/dO0fwjae/HFDQ6M9Dbx/B3J9vUnad7ezy4NF3j+LDw7UB17X0qKzIu9jKK4EyzAwrRETDiWGFopooinh1dzkOVlp7bW0/WFtPNaDd5cFzW84AAF7eUYa/fFWCh985HHCd3LOi8i3UkQ8ulCsrWQkGEBHR8GFYoajW7HApG63V2jrPcnXfvF5R2Sr/34drUdbkwPNfFAMA6mxO1Pu9tjwNlJsUC8C/siKvBGJlhYhoODGsUFTzPziw54nHg9HQ5oTLI4Uej1fE91/cjTpb97k/h6u7qzZyg628t4pSWfF9f/asEBENL4YVimpVLd1hpe4cKiuVvteR92ArbnAAAOJ0agDAoUqbcq3cszLeF1ZsnW643N7uBlszp4GIiIYTwwpFtYDKis3Zz5UDe515uYnK9E5CrBa3L5c2cztU5V9ZkaZ98pLioPE1rpxpaFMCz7jU+CGPg4iIemNYoahW6V9ZGcQ0UGeXB//vrcPYerIBQHe/SU5SLH56yUQAwE8unohF45IBIKB5V26wTYrTIilOBwD4+EgdAGB8apzyGBERDQ9uCkdRLbCyMvCwsnFfFf62owy7Sppx4aRUZTopM8GAb83LxqXT02EyaOFwuiEI0ms32J1INeqVnpWEWB2S4/Wotzvx0RFpefNATlMmIqLBYWWFolrPnhVRFAd035enpYrK6YY2dHZ5lNCTlSBNAZkM0inKcXoNxvumdeTqSmuHFFYSY3VIiZeqKEdrpJ6W+fmJ5/TzEBFRbwwrFNX8KytOtxfWjq5+rpZ4vCK2nWlSPj9d36aEnqzE3it5ZmZJBxIeqrLC6xWVnpXEWC1S4gPP/Jmfz8oKEdFwY1ihqNXmdCvhJEYrrdoZyFTQkWprwPk+R2tsfpWV3mFlhl9YsXe64fUVbxJidUj2609JidchPzl2aD8MERH1iWGFopbcFGsyaJDnCwkD2Wvli1ONAV/vLG5Gm9MNIHhYkSsrh6usSnNtnE4NnUaFZL/Kyvy8JAjy2mciIho2DCsUtbqnbmJh8e1tMpC9Vr46LYUVOYRsPlEPAEiO0yHGt6+Kv+mZJggCUGPtxDFfb0pCrFRRSY7vrqywX4WIKDQYVihqVfpN3VhMUliptQbfa+XJTSdx5dNfYvOJeuwpbQEA3HLhOADSlv1A8H4VQGqynZ8nBZFXdpUDkPZgAaA02ALsVyEiChWGFYoqZU0OXP3MV9h0tE6prGQnxiBdDitBKivFDW34/WencKDSiu9v2A2Xx4tMswErpqVDreqetsns57TkK2ZlAuieQkr0VVZS46Xva9CqMD3TNAw/IRER9cSwQlHl7aJq7CtvxYNvHkJJYxsAX2WlxzSQy+1V7nn+i2KIIgI2aztvQgoMWjXGp8Ypj/VVWQGAVTMs8G9HkSsr0zJNuH5hLv7fFdOgVfP/TkREocBN4SiqVLa0AwDq7U5sOirtGpuZEINYX69JrbUT+8pbcN0fd+DiKWm4f9UUvLG3CgDwp5vmoaq1A6/vrcT3zysAAEzNMOFkXXfo6UuayYBFBUnYUdwMoLuyolYJWH/1zBD8pEREJGNYoajiv72+vIQ4KzEGOl9Vo87Wiee3FsPl8eLDI7XYeqoBLo8XC/ITMT8/CfMBXDknS3mNqRkmvF1UrbxOf66YlamEFbmyQkREoce6NYXNxn2V+NeeikHdU+GrrPhPyWQlxCDdJC0hbnK4lIqL0aBBu8sDAMqBhD1NzejuM+mvsgJIU0Fyi4u8GoiIiEKPYYXCwuF0479eP4j73jiIFt9qHAC9tst3ub3o8kj9J26PFzWtUk/KfyzKAwDoNSqkxOuQFKdTqitur4i5uQl4+8fnYV5eIi6fmYGLJqcFHcc0v7CSfZbKSnK8XnmdcX69LkREFFqcBqKwqLF2wOObxzlV34aFBUn44lQDbv3bXvz8yhn41rxsuD1efOvZbWhsc+LTe5ahpb0Lbq8IrVrAvZdNRkmjAzOyzMpGbGkmvTJNdP3CXIxLjccbty/tdxypRj3uXzUFwMCqJU9+Zw72V7Rg2aTUc/nxiYhoEBhWKCyqW7uXGJ+qt2NhQRLePVCNdpcHv/roOFbPzsSmo3U45Ds88FClVbk+MyEG5hgtXr55UcBrWkwGVLZ0wGjQKEuNB+K2ZcGniIIxx2qxvI8qDRERhQangSgsaqzdjbKnfKtxjtXYAQB1NifeO1iNF74sVq45VmNDha9qkpMY/PwdeRrn6sKsoDvREhFRdGJlhcKixu8Mn9P1bXB7vDhRZ1ceW//v42iwd+9Ge7TGhkxfA2xfvSVrL56ANJMBd/TRTEtERNGJYYXCoqbHNFBJowMut1c5PVkOKukmPepsThyrsStLlfsKKxPSjPifr08N7cCJiGjEhXUaKD8/H4IgBHzcf//94RwSjZBqv2mgOpsTO0qk/UumZZpwzfxs5bl1q6cDAE7U2VHa6AAAZPcxDURERKNT2Csrjz76KG655Rbl6/j4+DCOhkaK/zQQALzr25htaoYRt1wwDu8drMGS8cm4bLoFcTo1HC4PiipaAQA5Sf0vMSYiotEl7GHFaDTCYrGEexg0wmp9YSUrIQZVrR3YVSpVVqZmmJCTFIu9P/saAEAQBEzJMGFvWQvcvnkgVlaIiMaWsK8G+uUvf4nk5GTMmTMHjz32GFwuV7/XO51O2Gy2gA+KLrbOLrQ53QCACyamBDwn7ygrTwsCgRu36dQqpMbrR2ikREQUCcIaVn7605/i1VdfxebNm7F27Vo89dRTuOOOO/q9Z/369TCbzcpHTk7OCI2WhovcXGuO0WJWdoLyuCAAUyzGXtcHbImfGAOVSuh1DRERjV7DHlbWrVvXq2m258eePXsAAHfddReWLVuGWbNm4eabb8Zzzz2HF154AU1NTX2+/gMPPACr1ap8VFQM7mwZGlkOpxvFDW0Bj8nNtRlmAyamd/coFSTHIVbXe2ZyakZ3gDnblvhERDT6DHvPytq1a3Hdddf1e01+fn7QxxcvXgwAOH36NJKTk4Neo9froddzGiAauD1eXPPcdhytseHWC8fhvpVToFYJSr9KZkIMJqR2hxX/Coq/KRYTVIJ0yjL7VYiIxp5hDyspKSlISUk5+4VB7N+/HwCQkZExnEOiMHllVzmO1kg9RX/cWoxT9W34/fWFqGmVKisWswGJcTqkxOvR2OYMqKD4i9GpkZ8Sh+IGBysrRERjUNh6VrZv347f/va3KCoqQklJCf75z3/i1ltvxTe+8Q3k5uaGa1g0TFrbXXhy00kAwDdmZ0KvUeGz4/X43WenUC1XVswGAMCicUkAgCXjg1fTAODCidLBgXNzE0M5bCIiikBhW7qs1+vx2muv4ZFHHoHT6UReXh5uueUW3HfffeEaEg2jpz45hdb2LkxON+LJ78zGR0cs+PEr+/DG3iqMT40DAGSYpSrJ+qtnYu1FE/qcBgKAh66YhjuWj0eayTAi4yciosgRtrAyd+5c7NixI1zfnkKopNGBv+0oAwA8tHoaNGoVVkxPR0q8Do1tTjQ5pK30MxKk4GEyaGHK0Pb7miqVwKBCRDRGhX2fFRp9nvj4BDxeERdPScN5E6T+Ja1ahavmZAEARN8ZP3JlhYiIqD8MKzSsjlRb8d7BGgDAvSsmBzx3zfzAPXEyzKyUEBHR2TGs0LD6zUcnAEhNtdMyA3tQJluMmJVtBgAkxelg8J2wTERE1B+GFRo2+8tbsPlEAzQqAXdfOinoNdfMk05UzuESZCIiGqCwH2RIo8eHR2oBAJfPykB+SlzQa65dkIsmh0vpZSEiIjobVlZoQGqsHbj5pd349Fhdn9d8eaoRAHDR5LQ+r9FpVLjza5OwID9p2MdIRESjE8PKGCGKIjYdrUNlS/uQ7n9jbyU+OVaP21/eh71lzb2eb2pz4ki1tFstqyZERDScGFbGiH3lLbjlr3tw9z8PDOn+k3XSYYQujxc/+utelDcFhp6vzkiHT06xGJFq5NlNREQ0fBhWxojSRilcHKmyQpQ3OhmEk3V2AIDJoEGTw4UbX9iBw1VW5fkvTjYAAC6YyKoKERENL4aVMaLZ4QIAOFwe1Nqks3l2lTTjsfePwuF093uv2+NFcYMDAPDC9xYgJykGFc0duPrZbXhtdzlEUcSXp6V+lfN9Z/gQERENF4aVMaLJF1YA4Ey9FDx+/t5RPP9FCZ7bcqbfe8ua2+HyeBGjVWNebiLeXXs+Lp6SBpfbi/9+4xBufmkPaqyd0KlVWMjGWSIiGmYMK2NEU5tT+fx0vR0utxfHa6WG2Be3lcLe2dXnvad8/SoT0uKhUglIiNXhz9+dj/+6bDJUAvDp8XoAwPz8RMTouNEbERENL4aVMaLZv7LS4MDJOju6PFLvir3TjZd3lPd57ylfv8rEtHjlMZVKwI8vmoCXb16ElHgdAODiKX0vWSYiIhoqhpUxwn8a6HR9G45US82xBq30K/DCl8Xo7PIo17Q53UrAOVkvVVYmpht7ve7S8Sn4908vxP9dNwf/sTgvZOMnIqKxi2FljGhydE8DnWlow+EqaQrohoV5yE6MQWObC6/slKorHS4PrvjdF1j+682otXYqlZVJ6fG9XxhAqlGPK+dk8awfIiIKCYaVMaK5rbuyUm93YnuxtC/KnNwE3L58PADgqU9OorHNiT9tLUZpUztsnW688GWxshJoUpDKChERUagxrESx7Wea8J3ntqO4oS3g8Z77qHR2eeBwSVM8Rr10HNRp39TOjEwTrp2fg+mZJtg63fjv1w8GrA56cVupshIoK4GHDxIR0chjWIliG74qwa7SZqU51un2YOVTW/GtZ7fB6+0OLHK/ilYtYGa2WXk8TqdGfnIcNGoVfn7VDADSyp6OLg/m5SViXGqc0oQrrwQiIiIaaQwrUey0r6Ii7yR7uMqK47V27Ctvxe7S7vN75CmgpDgdJvit6JmeaVYCyNzcRFy/MEd57qErpuGWC8YpX0/so1+FiIgo1BhWolSXx6ucz3Ok2gqPV8T+8lbl+XcPViufN/qaa5Pj9IFhJcsU8Jr/vXIKlk1KxV1fm4TZOQn4ZmGWsiyZ/SpERBQuDCtRqqzJAbdvqsfh8qCksQ37K1qV5/99qBZujxdAd2UlOV6H8andYWVGZveUEAAkxOrw0g8W4qdfmwgAMGjV+N+rZmBhQRKumpMVyh+HiIioT5pwD4CGRm6QlR2qsqLIr7LS5HBhe3ETLpiYquyX0nMaaEZWYFgJZuWMDKyckTE8gyYiIhoCVlai1BnfcmLZZ8cbUNXaAUEArpyTCQB494A0FeQ/DZRm1OPrMy24aHJqQHAhIiKKVAwrUUqurIxPjQMAfHi4BoC0Jf71C3N9j9XC5fYGTAMJgoBnbpyHDd9fCDVX9xARURRgWIlScli5em42AChLjAtzErEgPwlpRj1snW7sLGlSpoGS43ThGSwREdE5YFiJQqIo4oxv2fKl09IR63fS8ZzcBKhVApaOTwYA7C1rQaNfzwoREVG0YViJQjXWTrS7PNCoBBSkxGF6ZvcS5MLcBADAvLxEAFJYaZZ7VuIZVoiIKPowrIRYh8uDn/xjP97aXzVsrylPAeUlx0KrVmFmVgIAaUfaiWnSfiiFuVJYKSpvRaNdngbSD9sYiIiIRgrDSoh9fLQW7xyoxhObTgzba8pTQPJqnoUFUjBZUJCkNM1OsRgRq1PD7nSjo0s6FyiJlRUiIopC3GclxOSt8CtbOtDZ5YFBqz7LHWfXvRJICiuXTbfgDzfMxdy8BOUajVqFWdlm7CiWtt3XqVXKIYZERETRhJWVEDvkCyui2F0ROVdyWJErK4Ig4PJZGcgwB56KLPetAFJzrSBwqTIREUUf/qd2CHm9Io5U2ZSvT9e3YXqmGa3tLmw/0wRBEGA0aDA/PxF6zcAqLl6viOO1dgBQ+lP6Mjc3MKwQERFFI4aVECpvbofd6Va+lisi971+EB8frVMen5ZhwjM3zkV+StxZX/NYrQ3Wji7E6dSYktF/WCn0CytcCURERNGK00AhJE8ByU7Xt8Ht8eKr040AgFnZZphjtDhaY8MVv/8Sm0/Un/U1t59pAiA102rV/f/xJcXpUOALQNwQjoiIolVIw8pjjz2GpUuXIjY2FgkJCUGvKS8vx+rVqxEXF4eUlBT85Cc/gcvlCuWwRszhaims5CRJvSSn69twvNYOh8sDo0GDN+84Dx/deSEW5CeizenG/2w8pNz74eEafPvZbThdbw94zR3FUlhZMi55QGOQp4JS4rlsmYiIolNIw4rL5cI111yD22+/PejzHo8Hl19+ORwOB7788ku8+uqreOONN3DPPfeEclgjRl4JdNWcLABAaZNDqYzMy0uEWiXAYjbgxe8vhEqQNnurt3UCAP60tRh7ylrws7cOQxSlrfQ9XhE7S6TVPUvGDyys3HJhAb4+04JrF+QM689GREQ0UkIaVh555BHcddddmDlzZtDnP/74Yxw9ehQvv/wyCgsL8bWvfQ1PPPEEnn/+edhstqD3RAtRFHHY11y7YpoFsTo1ujwi3thXCQBYkJ+kXBun1yjLkA9XW9Hl8eJwtXTvjuJmbPL1txyptsLe6YbRoMH0TPOAxjHFYsIzN87DxPT++1uIiIgiVVh7VrZv344ZM2YgMzNTeeyyyy6D0+nE3r17g97jdDphs9kCPiJRZUsHrB1d0KoFTLYYlTAir+TxDysAMCNLCh+HKm04UWuHy+1Vnlv/7+Nwub1KVWaR3+ZvREREo11Yw0ptbS3S09MDHktMTIROp0NtbW3Qe9avXw+z2ax85ORE5vSGPAU02WKETqNS9kQBpA3aZmUHVkbksHK42oqiilYA0jk/KfE6lDQ68PA7h/HpcakBd/EA+1WIiIhGg0GHlXXr1kEQhH4/9uzZM+DXC7ZRmSiKfW5g9sADD8BqtSofFRUVg/0RRsQR3zTODN90jX9YmZlt7rWT7Uw5rFRZccAXVs4bn4L7Vk4BAPxjVwV2DbJfhYiIaDQY9D4ra9euxXXXXdfvNfn5+QN6LYvFgp07dwY81tLSgq6url4VF5ler4deH/krW8qb2wEA41KlpcPyNBAAzM9P7HX9tEwTBF+T7dZTDQCA2TkJuHRaOpLjdHji45M4WmNDptmAqRZTr/uJiIhGq0GHlZSUFKSkpAzLN1+yZAkee+wx1NTUICMjA4DUdKvX6zFv3rxh+R7hUtXaAQDISogFEFhZWZCX1Ov6eL0GBSlxKG5woM7mBADM9k0VXTI1HRdNTsOeshakm/RQsV+FiIjGkJDuYFteXo7m5maUl5fD4/GgqKgIADBhwgTEx8djxYoVmDZtGm666Sb8+te/RnNzM+69917ccsstMJmiu3pQ1eILK4nSHit5ybFIidfD2eXp1Vwrm5llRnGDAwCQaTYgzWRQnlOpBCwsCH4fERHRaBbSsPLQQw/hpZdeUr4uLCwEAGzevBnLly+HWq3G+++/jzvuuAPnnXceYmJicMMNN+A3v/lNKIcVci63F3V2ab+UbF9Y0apVeOP2JejyiDDHaoPeNyPTjLeLqgFIU0BEREQU4rDy4osv4sUXX+z3mtzcXLz33nuhHMaIq7F2QBQBg1YVsM19XnL/Z//IK4IAYFZ2QqiGR0REFFV4NlAIyFNAmQkxfa5qCmZ6VvfU1+ycgW36RkRENNrx1OUh6m95daXSXBszqNc0GbS4bkEOShodypk+REREYx3DyhC8XVSFde8cwR9umIulE3qvjKr0VVbkfpXB+MW3Zp3z+IiIiEYTTgMNwWfH69HS3oX3DtUEfb5KCSuxIzksIiKiUYlhZQha2rsAdO9S21NVq7Qh3GCngYiIiKg3hpUhaHG4AADHa2xwe7y9nlc2hBvCNBAREREFYlgZgmZfWHG6vTjj28RN5vGKqGmV9lhhZYWIiOjcMawMQWu7S/lcPl1ZVmfrhNsrQqMSkO63Ay0RERENDcNKH7aebMAPXtyNv3xZEvB4Z5cHDpdH+bpn34o8BZSRYICaZ/gQERGdM4aVPpQ3t+Oz4/V4+0B1wOOtvuZa2ZHqwMqKciYQp4CIiIiGBcNKHy6bboEgAAcqWlHZ0q483uI3BQQAR6tt8HpF5euepy0TERHRuWFY6UOqUY+FvtORPzxcqzwurwQqSImDTqOC3elGhV+YkYMNVwIREREND4aVfqyaYQEQGFaafZWV1Hg9pliMAAL7Vs5l91oiIiLqjWGlHytnZAAA9pS1oNYqLUeWN4RLjNNieqZ08KD/iqDT9W0AgLwkTgMRERENB4aVfljMBszLkw4U/OiIVF2Rp4GS4nSYnimdjHzIF1aqWztQY+2EWiVgZjZPTSYiIhoODCtnIU8FfeA7B0jeEC4hVof5+VKQ2VPaAqfbg71lLQCAaRkmxOp4RiQREdFwYFg5i0unpQMA9pa1wO3xKhvCJcXqMDndiJR4HTq6PNhf3qqEFbkaQ0REROeOYeUschJjodOo4PaKqG7tRLPSs6KDIAg4b0IKAOCr043YXy6FlbkMK0RERMOGYeUsVCpBaZYta3YoPSuJsVoAUMLKpqN1yqogVlaIiIiGD8PKAOQlS2GltKld6VlJjNMB6A4rx2vtcHtFWEwGZJp5JhAREdFwYVgZgLzkOABAWaMjoGcFkLbVH5cSp1w7Ly8RgsAzgYiIiIYLw8oA5PsqKyfq7MohhnJlBeiurgBAYW7CiI6NiIhotGNYGQC5snKgohUAoFYJMBm6lyb7hxX2qxAREQ0vbgYyAPm+sGLrdAOQmmv9p3qWjE+GOUYLvUalbBRHREREw4NhZQAyEwzQqAS4facrJ8bqAp43x2jx/k/Oh1olQKdhsYqIiGg48V/WAdCoVQEHE/YMKwCQnRiLDDMPLyQiIhpuDCsDJPetANIhhkRERDQyGFYGSF4RBEiHGBIREdHIYFgZIP/KSkKQaSAiIiIKDYaVAcpP8ausMKwQERGNGIaVAcpN8u9ZYVghIiIaKQwrA5STFAN5axX5EEMiIiIKPYaVAdJr1Cjw9a1wiTIREdHI4aZwg/B/1xXiRJ0dUzOM4R4KERHRmBHSyspjjz2GpUuXIjY2FgkJCUGvEQSh18dzzz0XymEN2cxsM749L5unKhMREY2gkFZWXC4XrrnmGixZsgQvvPBCn9dt2LABK1euVL42m3m+DhEREUlCGlYeeeQRAMCLL77Y73UJCQmwWCyhHAoRERFFqYhosF27di1SUlKwYMECPPfcc/B6vX1e63Q6YbPZAj6IiIho9Ap7g+3Pf/5zXHLJJYiJicGnn36Ke+65B42NjfjZz34W9Pr169crFRsiIiIa/QZdWVm3bl3Qplj/jz179gz49X72s59hyZIlmDNnDu655x48+uij+PWvf93n9Q888ACsVqvyUVFRMdgfgYiIiKLIoCsra9euxXXXXdfvNfn5+UMdDxYvXgybzYa6ujqkp6f3el6v10Ov1w/59YmIiCi6DDqspKSkICUlJRRjAQDs378fBoOhz6XORERENLaEtGelvLwczc3NKC8vh8fjQVFREQBgwoQJiI+Px7vvvova2losWbIEMTEx2Lx5Mx588EH86Ec/YvWEiIiIAIQ4rDz00EN46aWXlK8LCwsBAJs3b8by5cuh1WrxzDPP4O6774bX68W4cePw6KOP4sc//nEoh0VERERRRBBFUQz3IM6FzWaD2WyG1WqFyWQK93CIiIhoAAbz73dE7LNCRERE1BeGFSIiIopoDCtEREQU0cK+g+25kltuuO0+ERFR9JD/3R5I62zUhxW73Q4AyMnJCfNIiIiIaLDsdjvMZnO/10T9aiCv14vq6moYjUYIghDu4ZwTm82GnJwcVFRUjOmVTXwfJHwf+B7I+D7wPZCNpvdBFEXY7XZkZmZCpeq/KyXqKysqlQrZ2dnhHsawMplMUf9LOBz4Pkj4PvA9kPF94HsgGy3vw9kqKjI22BIREVFEY1ghIiKiiMawEkH0ej0efvjhMX8uEt8HCd8Hvgcyvg98D2Rj9X2I+gZbIiIiGt1YWSEiIqKIxrBCREREEY1hhYiIiCIawwoRERFFNIaVMHvsscewdOlSxMbGIiEhYUD3fO9734MgCAEfixcvDu1AQ2wo74Moili3bh0yMzMRExOD5cuX48iRI6EdaAi1tLTgpptugtlshtlsxk033YTW1tZ+7xkNvwvPPPMMCgoKYDAYMG/ePHzxxRf9Xr9lyxbMmzcPBoMB48aNw3PPPTdCIw2dwbwHn3/+ea8/c0EQcPz48REc8fDbunUrVq9ejczMTAiCgLfeeuus94y234XBvgej9XchGIaVMHO5XLjmmmtw++23D+q+lStXoqamRvn44IMPQjTCkTGU9+FXv/oVnnzySTz99NPYvXs3LBYLLr30UuW8qGhzww03oKioCB9++CE+/PBDFBUV4aabbjrrfdH8u/Daa6/hzjvvxIMPPoj9+/fjggsuwKpVq1BeXh70+pKSEnz961/HBRdcgP379+N//ud/8JOf/ARvvPHGCI98+Az2PZCdOHEi4M994sSJIzTi0HA4HJg9ezaefvrpAV0/Gn8XBvseyEbb70JQIkWEDRs2iGazeUDXrlmzRrzyyitDOp5wGej74PV6RYvFIv7iF79QHuvs7BTNZrP43HPPhXCEoXH06FERgLhjxw7lse3bt4sAxOPHj/d5X7T/LixcuFC87bbbAh6bMmWKeP/99we9/r777hOnTJkS8Nitt94qLl68OGRjDLXBvgebN28WAYgtLS0jMLrwACC++eab/V4zGn8X/A3kPRgLvwsyVlai1Oeff460tDRMmjQJt9xyC+rr68M9pBFVUlKC2tparFixQnlMr9dj2bJl2LZtWxhHNjTbt2+H2WzGokWLlMcWL14Ms9l81p8nWn8XXC4X9u7dG/BnCAArVqzo82fevn17r+svu+wy7NmzB11dXSEba6gM5T2QFRYWIiMjA5dccgk2b94cymFGpNH2u3AuxsLvAsNKFFq1ahX+/ve/47PPPsMTTzyB3bt34+KLL4bT6Qz30EZMbW0tACA9PT3g8fT0dOW5aFJbW4u0tLRej6elpfX780Tz70JjYyM8Hs+g/gxra2uDXu92u9HY2BiysYbKUN6DjIwM/OlPf8Ibb7yBjRs3YvLkybjkkkuwdevWkRhyxBhtvwtDMZZ+F6L+1OVItG7dOjzyyCP9XrN7927Mnz9/SK9/7bXXKp/PmDED8+fPR15eHt5//31cffXVQ3rNUAj1+wAAgiAEfC2KYq/Hwmmg7wHQ+2cBzv7zRMvvQn8G+2cY7Ppgj0eTwbwHkydPxuTJk5WvlyxZgoqKCvzmN7/BhRdeGNJxRprR+LswGGPpd4FhJQTWrl2L6667rt9r8vPzh+37ZWRkIC8vD6dOnRq21xwOoXwfLBYLAOm/rjIyMpTH6+vre/3XVjgN9D04ePAg6urqej3X0NAwqJ8nUn8XgklJSYFare5VQejvz9BisQS9XqPRIDk5OWRjDZWhvAfBLF68GC+//PJwDy+ijbbfheEyWn8XGFZCICUlBSkpKSP2/ZqamlBRURHwj3YkCOX7UFBQAIvFgk2bNqGwsBCANP+/ZcsW/PKXvwzJ9xyKgb4HS5YsgdVqxa5du7Bw4UIAwM6dO2G1WrF06dIBf79I/V0IRqfTYd68edi0aRO++c1vKo9v2rQJV155ZdB7lixZgnfffTfgsY8//hjz58+HVqsN6XhDYSjvQTD79++Pij/z4TTafheGy6j9XQhndy+JYllZmbh//37xkUceEePj48X9+/eL+/fvF+12u3LN5MmTxY0bN4qiKIp2u1285557xG3btoklJSXi5s2bxSVLlohZWVmizWYL149xzgb7PoiiKP7iF78QzWazuHHjRvHQoUPi9ddfL2ZkZETt+7By5Upx1qxZ4vbt28Xt27eLM2fOFK+44oqAa0bb78Krr74qarVa8YUXXhCPHj0q3nnnnWJcXJxYWloqiqIo3n///eJNN92kXF9cXCzGxsaKd911l3j06FHxhRdeELVarfj666+H60c4Z4N9D37729+Kb775pnjy5Enx8OHD4v333y8CEN94441w/QjDwm63K/+/ByA++eST4v79+8WysjJRFMfG78Jg34PR+rsQDMNKmK1Zs0YE0Otj8+bNyjUAxA0bNoiiKIrt7e3iihUrxNTUVFGr1Yq5ubnimjVrxPLy8vD8AMNksO+DKErLlx9++GHRYrGIer1evPDCC8VDhw6N/OCHSVNTk3jjjTeKRqNRNBqN4o033thrSeJo/F34wx/+IObl5Yk6nU6cO3euuGXLFuW5NWvWiMuWLQu4/vPPPxcLCwtFnU4n5ufni88+++wIj3j4DeY9+OUvfymOHz9eNBgMYmJionj++eeL77//fhhGPbzkZbg9P9asWSOK4tj4XRjsezBafxeCEUTR15FEREREFIG4dJmIiIgiGsMKERERRTSGFSIiIopoDCtEREQU0RhWiIiIKKIxrBAREVFEY1ghIiKiiMawQkRERBGNYYWIiIgiGsMKERERRTSGFSIiIopoDCtEREQU0f4/1YSN4IwWlAsAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(x,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y = 2.82 * x + -1.91\n",
      "y = 2.77 * x + -1.81\n",
      "y = 2.50 * x + -0.95\n",
      "y = 2.84 * x + 0.55\n",
      "y = 4.34 * x + 2.48\n",
      "y = 7.04 * x + 4.50\n",
      "y = 8.87 * x + 5.61\n",
      "y = 9.22 * x + 5.28\n",
      "y = 9.29 * x + 5.14\n",
      "y = 9.30 * x + 5.14\n",
      "y = 9.30 * x + 5.13\n",
      "y = 9.47 * x + 5.27\n",
      "y = 9.75 * x + 5.45\n",
      "y = 9.88 * x + 5.34\n",
      "y = 9.94 * x + 5.25\n",
      "y = 9.95 * x + 5.23\n",
      "y = 9.89 * x + 5.10\n",
      "y = 9.89 * x + 5.10\n",
      "y = 9.99 * x + 5.16\n",
      "y = 9.98 * x + 5.16\n"
     ]
    }
   ],
   "source": [
    "model = Linear()\n",
    "\n",
    "batch_size = 32\n",
    "lr = 0.1\n",
    "\n",
    "for t in range(20):\n",
    "    ix = (t * batch_size) % len(x)\n",
    "    xx = x[ix:ix+batch_size]\n",
    "    yy = y[ix:ix+batch_size]\n",
    "    loss = mse([model.error(_x,_y) for _x,_y in zip(xx,yy)])\n",
    "    loss.backward()\n",
    "    model.a -= lr * model.a.grad\n",
    "    model.b -= lr * model.b.grad\n",
    "    model.a.grad = 0\n",
    "    model.b.grad = 0\n",
    "\n",
    "    print(model.string())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 计算图膨胀"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "(process:24036): Pango-WARNING **: 13:43:44.706: couldn't load font \"Menlo Not-Rotated 10\", falling back to \"Sans Not-Rotated 10\", expect ugly output.\n",
      "\n",
      "(process:24036): Pango-WARNING **: 13:43:44.713: couldn't load font \"Menlo Not-Rotated 14\", falling back to \"Sans Not-Rotated 14\", expect ugly output.\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.50.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"320pt\" height=\"502pt\"\n",
       " viewBox=\"0.00 0.00 320.20 502.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 498)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-498 316.2,-498 316.2,4 -4,4\"/>\n",
       "<!-- 1639796495888backward -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>1639796495888backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M180,-109.5C180,-109.5 222,-109.5 222,-109.5 228,-109.5 234,-115.5 234,-121.5 234,-121.5 234,-154.5 234,-154.5 234,-160.5 228,-166.5 222,-166.5 222,-166.5 180,-166.5 180,-166.5 174,-166.5 168,-160.5 168,-154.5 168,-154.5 168,-121.5 168,-121.5 168,-115.5 174,-109.5 180,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"201\" y=\"-154.5\" font-family=\"Menlo\" font-size=\"10.00\">grad=&#45;1.50</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"168,-147.5 234,-147.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"201\" y=\"-135.5\" font-family=\"Menlo\" font-size=\"10.00\">value=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"168,-128.5 234,-128.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"201\" y=\"-116.5\" font-family=\"Menlo\" font-size=\"10.00\">*</text>\n",
       "</g>\n",
       "<!-- 1639796830288backward -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>1639796830288backward</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"201\" cy=\"-29\" rx=\"31.4\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"201\" y=\"-26.5\" font-family=\"Menlo\" font-size=\"10.00\">x1=1.00</text>\n",
       "</g>\n",
       "<!-- 1639796495888backward&#45;&gt;1639796830288backward -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>1639796495888backward&#45;&gt;1639796830288backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M201,-109.44C201,-89.75 201,-63.88 201,-47.09\"/>\n",
       "</g>\n",
       "<!-- 1639796165840backward -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>1639796165840backward</title>\n",
       "<polygon fill=\"lightgreen\" stroke=\"black\" stroke-width=\"2\" points=\"84,-0.5 84,-57.5 150,-57.5 150,-0.5 84,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-45.5\" font-family=\"Menlo\" font-size=\"10.00\">grad=&#45;9.50</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"84,-38.5 150,-38.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-26.5\" font-family=\"Menlo\" font-size=\"10.00\">value=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"84,-19.5 150,-19.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-7.5\" font-family=\"Menlo\" font-size=\"10.00\">a</text>\n",
       "</g>\n",
       "<!-- 1639796495888backward&#45;&gt;1639796165840backward -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>1639796495888backward&#45;&gt;1639796165840backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M178.6,-109.34C173.76,-103.32 168.69,-96.96 164,-91 157.61,-82.89 150.8,-74.11 144.4,-65.82\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"147.04,-63.52 138.17,-57.73 141.5,-67.79 147.04,-63.52\"/>\n",
       "<text text-anchor=\"middle\" x=\"180.5\" y=\"-79.8\" font-family=\"Menlo\" font-size=\"14.00\">&#45;1.50</text>\n",
       "</g>\n",
       "<!-- 1639796495376backward -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>1639796495376backward</title>\n",
       "<polygon fill=\"lightgreen\" stroke=\"black\" stroke-width=\"2\" points=\"84,-109.5 84,-166.5 150,-166.5 150,-109.5 84,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-154.5\" font-family=\"Menlo\" font-size=\"10.00\">grad=&#45;5.50</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"84,-147.5 150,-147.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-135.5\" font-family=\"Menlo\" font-size=\"10.00\">value=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"84,-128.5 150,-128.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-116.5\" font-family=\"Menlo\" font-size=\"10.00\">b</text>\n",
       "</g>\n",
       "<!-- 1639796496400backward -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>1639796496400backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M96,-327.5C96,-327.5 138,-327.5 138,-327.5 144,-327.5 150,-333.5 150,-339.5 150,-339.5 150,-372.5 150,-372.5 150,-378.5 144,-384.5 138,-384.5 138,-384.5 96,-384.5 96,-384.5 90,-384.5 84,-378.5 84,-372.5 84,-372.5 84,-339.5 84,-339.5 84,-333.5 90,-327.5 96,-327.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-372.5\" font-family=\"Menlo\" font-size=\"10.00\">grad=4.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"84,-365.5 150,-365.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-353.5\" font-family=\"Menlo\" font-size=\"10.00\">value=4.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"84,-346.5 150,-346.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-334.5\" font-family=\"Menlo\" font-size=\"10.00\">&#45;</text>\n",
       "</g>\n",
       "<!-- 1639796496528backward -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>1639796496528backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M14,-218.5C14,-218.5 56,-218.5 56,-218.5 62,-218.5 68,-224.5 68,-230.5 68,-230.5 68,-263.5 68,-263.5 68,-269.5 62,-275.5 56,-275.5 56,-275.5 14,-275.5 14,-275.5 8,-275.5 2,-269.5 2,-263.5 2,-263.5 2,-230.5 2,-230.5 2,-224.5 8,-218.5 14,-218.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"35\" y=\"-263.5\" font-family=\"Menlo\" font-size=\"10.00\">grad=&#45;4.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2,-256.5 68,-256.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"35\" y=\"-244.5\" font-family=\"Menlo\" font-size=\"10.00\">value=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"2,-237.5 68,-237.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"35\" y=\"-225.5\" font-family=\"Menlo\" font-size=\"10.00\">+</text>\n",
       "</g>\n",
       "<!-- 1639796496400backward&#45;&gt;1639796496528backward -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>1639796496400backward&#45;&gt;1639796496528backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M94.51,-327.42C89.67,-321.39 84.63,-315.01 80,-309 73.73,-300.87 67.11,-292.03 60.93,-283.67\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"63.67,-281.49 54.92,-275.51 58.03,-285.64 63.67,-281.49\"/>\n",
       "<text text-anchor=\"middle\" x=\"96.5\" y=\"-297.8\" font-family=\"Menlo\" font-size=\"14.00\">&#45;4.00</text>\n",
       "</g>\n",
       "<!-- 1639796323536backward -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>1639796323536backward</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"117\" cy=\"-247\" rx=\"31.4\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-244.5\" font-family=\"Menlo\" font-size=\"10.00\">y2=4.00</text>\n",
       "</g>\n",
       "<!-- 1639796496400backward&#45;&gt;1639796323536backward -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>1639796496400backward&#45;&gt;1639796323536backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M117,-327.44C117,-307.75 117,-281.88 117,-265.09\"/>\n",
       "</g>\n",
       "<!-- 1639796496528backward&#45;&gt;1639796495376backward -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>1639796496528backward&#45;&gt;1639796495376backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M56.12,-218.44C66.27,-205.19 78.57,-189.14 89.45,-174.94\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"92.36,-176.9 95.67,-166.83 86.81,-172.64 92.36,-176.9\"/>\n",
       "<text text-anchor=\"middle\" x=\"96.5\" y=\"-188.8\" font-family=\"Menlo\" font-size=\"14.00\">&#45;4.00</text>\n",
       "</g>\n",
       "<!-- 1639796496272backward -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>1639796496272backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M12,-109.5C12,-109.5 54,-109.5 54,-109.5 60,-109.5 66,-115.5 66,-121.5 66,-121.5 66,-154.5 66,-154.5 66,-160.5 60,-166.5 54,-166.5 54,-166.5 12,-166.5 12,-166.5 6,-166.5 0,-160.5 0,-154.5 0,-154.5 0,-121.5 0,-121.5 0,-115.5 6,-109.5 12,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"33\" y=\"-154.5\" font-family=\"Menlo\" font-size=\"10.00\">grad=&#45;4.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"0,-147.5 66,-147.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"33\" y=\"-135.5\" font-family=\"Menlo\" font-size=\"10.00\">value=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"0,-128.5 66,-128.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"33\" y=\"-116.5\" font-family=\"Menlo\" font-size=\"10.00\">*</text>\n",
       "</g>\n",
       "<!-- 1639796496528backward&#45;&gt;1639796496272backward -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>1639796496528backward&#45;&gt;1639796496272backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M34.37,-218.27C34.24,-212.25 34.11,-205.91 34,-200 33.86,-192.5 33.72,-184.48 33.59,-176.81\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"37.09,-176.58 33.42,-166.63 30.09,-176.69 37.09,-176.58\"/>\n",
       "<text text-anchor=\"middle\" x=\"50.5\" y=\"-188.8\" font-family=\"Menlo\" font-size=\"14.00\">&#45;4.00</text>\n",
       "</g>\n",
       "<!-- 1639796496080backward -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>1639796496080backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M178,-218.5C178,-218.5 220,-218.5 220,-218.5 226,-218.5 232,-224.5 232,-230.5 232,-230.5 232,-263.5 232,-263.5 232,-269.5 226,-275.5 220,-275.5 220,-275.5 178,-275.5 178,-275.5 172,-275.5 166,-269.5 166,-263.5 166,-263.5 166,-230.5 166,-230.5 166,-224.5 172,-218.5 178,-218.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"199\" y=\"-263.5\" font-family=\"Menlo\" font-size=\"10.00\">grad=&#45;1.50</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"166,-256.5 232,-256.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"199\" y=\"-244.5\" font-family=\"Menlo\" font-size=\"10.00\">value=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"166,-237.5 232,-237.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"199\" y=\"-225.5\" font-family=\"Menlo\" font-size=\"10.00\">+</text>\n",
       "</g>\n",
       "<!-- 1639796496080backward&#45;&gt;1639796495888backward -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>1639796496080backward&#45;&gt;1639796495888backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M199.52,-218.44C199.75,-205.81 200.03,-190.64 200.29,-176.95\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"203.79,-176.9 200.48,-166.83 196.79,-176.77 203.79,-176.9\"/>\n",
       "<text text-anchor=\"middle\" x=\"216.5\" y=\"-188.8\" font-family=\"Menlo\" font-size=\"14.00\">&#45;1.50</text>\n",
       "</g>\n",
       "<!-- 1639796496080backward&#45;&gt;1639796495376backward -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>1639796496080backward&#45;&gt;1639796495376backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M176.51,-218.42C171.67,-212.39 166.63,-206.01 162,-200 155.73,-191.87 149.11,-183.03 142.93,-174.67\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"145.67,-172.49 136.92,-166.51 140.03,-176.64 145.67,-172.49\"/>\n",
       "<text text-anchor=\"middle\" x=\"178.5\" y=\"-188.8\" font-family=\"Menlo\" font-size=\"14.00\">&#45;1.50</text>\n",
       "</g>\n",
       "<!-- 1639797063440backward -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>1639797063440backward</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"281\" cy=\"-247\" rx=\"31.4\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"281\" y=\"-244.5\" font-family=\"Menlo\" font-size=\"10.00\">y1=1.50</text>\n",
       "</g>\n",
       "<!-- 1639796496208backward -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>1639796496208backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M180,-327.5C180,-327.5 222,-327.5 222,-327.5 228,-327.5 234,-333.5 234,-339.5 234,-339.5 234,-372.5 234,-372.5 234,-378.5 228,-384.5 222,-384.5 222,-384.5 180,-384.5 180,-384.5 174,-384.5 168,-378.5 168,-372.5 168,-372.5 168,-339.5 168,-339.5 168,-333.5 174,-327.5 180,-327.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"201\" y=\"-372.5\" font-family=\"Menlo\" font-size=\"10.00\">grad=1.50</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"168,-365.5 234,-365.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"201\" y=\"-353.5\" font-family=\"Menlo\" font-size=\"10.00\">value=1.50</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"168,-346.5 234,-346.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"201\" y=\"-334.5\" font-family=\"Menlo\" font-size=\"10.00\">&#45;</text>\n",
       "</g>\n",
       "<!-- 1639796496208backward&#45;&gt;1639796496080backward -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>1639796496208backward&#45;&gt;1639796496080backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M200.48,-327.44C200.25,-314.81 199.97,-299.64 199.71,-285.95\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"203.21,-285.77 199.52,-275.83 196.21,-285.9 203.21,-285.77\"/>\n",
       "<text text-anchor=\"middle\" x=\"216.5\" y=\"-297.8\" font-family=\"Menlo\" font-size=\"14.00\">&#45;1.50</text>\n",
       "</g>\n",
       "<!-- 1639796496208backward&#45;&gt;1639797063440backward -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>1639796496208backward&#45;&gt;1639797063440backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M222.88,-327.41C227.58,-321.37 232.49,-315 237,-309 248.37,-293.86 260.98,-276.3 269.77,-263.92\"/>\n",
       "</g>\n",
       "<!-- 1639796497232backward -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>1639796497232backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M137,-436.5C137,-436.5 179,-436.5 179,-436.5 185,-436.5 191,-442.5 191,-448.5 191,-448.5 191,-481.5 191,-481.5 191,-487.5 185,-493.5 179,-493.5 179,-493.5 137,-493.5 137,-493.5 131,-493.5 125,-487.5 125,-481.5 125,-481.5 125,-448.5 125,-448.5 125,-442.5 131,-436.5 137,-436.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"158\" y=\"-481.5\" font-family=\"Menlo\" font-size=\"10.00\">grad=1.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"125,-474.5 191,-474.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"158\" y=\"-462.5\" font-family=\"Menlo\" font-size=\"10.00\">value=9.12</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"125,-455.5 191,-455.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"158\" y=\"-443.5\" font-family=\"Menlo\" font-size=\"10.00\">mse</text>\n",
       "</g>\n",
       "<!-- 1639796497232backward&#45;&gt;1639796496400backward -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>1639796497232backward&#45;&gt;1639796496400backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M147.44,-436.44C142.55,-423.69 136.67,-408.34 131.39,-394.54\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"134.51,-392.92 127.67,-384.83 127.98,-395.42 134.51,-392.92\"/>\n",
       "<text text-anchor=\"middle\" x=\"153\" y=\"-406.8\" font-family=\"Menlo\" font-size=\"14.00\">4.00</text>\n",
       "</g>\n",
       "<!-- 1639796497232backward&#45;&gt;1639796496208backward -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>1639796497232backward&#45;&gt;1639796496208backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M169.07,-436.44C174.25,-423.57 180.49,-408.04 186.07,-394.14\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"189.33,-395.42 189.81,-384.83 182.84,-392.81 189.33,-395.42\"/>\n",
       "<text text-anchor=\"middle\" x=\"196\" y=\"-406.8\" font-family=\"Menlo\" font-size=\"14.00\">1.50</text>\n",
       "</g>\n",
       "<!-- 1639796496272backward&#45;&gt;1639796165840backward -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>1639796496272backward&#45;&gt;1639796165840backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M54.63,-109.44C65.03,-96.19 77.64,-80.14 88.78,-65.94\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"91.73,-67.86 95.15,-57.83 86.22,-63.54 91.73,-67.86\"/>\n",
       "<text text-anchor=\"middle\" x=\"96.5\" y=\"-79.8\" font-family=\"Menlo\" font-size=\"14.00\">&#45;8.00</text>\n",
       "</g>\n",
       "<!-- 1639796422096backward -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>1639796422096backward</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"33\" cy=\"-29\" rx=\"31.4\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"33\" y=\"-26.5\" font-family=\"Menlo\" font-size=\"10.00\">x2=2.00</text>\n",
       "</g>\n",
       "<!-- 1639796496272backward&#45;&gt;1639796422096backward -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>1639796496272backward&#45;&gt;1639796422096backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M33,-109.44C33,-89.75 33,-63.88 33,-47.09\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.graphs.Digraph at 0x17dca5c5cd0>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 计算图膨胀\n",
    "model = Linear()\n",
    "# 定义两组数据\n",
    "x1 = Scalar(1.0,label=\"x1\",requires_grad=False)\n",
    "y1 = Scalar(1.5,label=\"y1\",requires_grad=False)\n",
    "x2 = Scalar(2.0,label=\"x2\",requires_grad=False)\n",
    "y2 = Scalar(4.0,label=\"y2\",requires_grad=False)\n",
    "\n",
    "loss = mse([model.error(x1,y1),model.error(x2,y2)])\n",
    "loss.backward()\n",
    "draw_graph(loss,\"backward\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "(process:19632): Pango-WARNING **: 13:49:05.493: couldn't load font \"Menlo Not-Rotated 10\", falling back to \"Sans Not-Rotated 10\", expect ugly output.\n",
      "\n",
      "(process:19632): Pango-WARNING **: 13:49:05.498: couldn't load font \"Menlo Not-Rotated 14\", falling back to \"Sans Not-Rotated 14\", expect ugly output.\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.50.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"224pt\" height=\"611pt\"\n",
       " viewBox=\"0.00 0.00 223.67 611.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 607)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-607 219.67,-607 219.67,4 -4,4\"/>\n",
       "<!-- 1639808597520backward -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>1639808597520backward</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"37.67\" cy=\"-465\" rx=\"37.85\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"37.67\" y=\"-462.5\" font-family=\"Menlo\" font-size=\"10.00\">input=0.50</text>\n",
       "</g>\n",
       "<!-- 1639796857424backward -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>1639796857424backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M64.67,-218.5C64.67,-218.5 106.67,-218.5 106.67,-218.5 112.67,-218.5 118.67,-224.5 118.67,-230.5 118.67,-230.5 118.67,-263.5 118.67,-263.5 118.67,-269.5 112.67,-275.5 106.67,-275.5 106.67,-275.5 64.67,-275.5 64.67,-275.5 58.67,-275.5 52.67,-269.5 52.67,-263.5 52.67,-263.5 52.67,-230.5 52.67,-230.5 52.67,-224.5 58.67,-218.5 64.67,-218.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"85.67\" y=\"-263.5\" font-family=\"Menlo\" font-size=\"10.00\">grad=&#45;1.50</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"52.67,-256.5 118.67,-256.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"85.67\" y=\"-244.5\" font-family=\"Menlo\" font-size=\"10.00\">value=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"52.67,-237.5 118.67,-237.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"85.67\" y=\"-225.5\" font-family=\"Menlo\" font-size=\"10.00\">+</text>\n",
       "</g>\n",
       "<!-- 1639796325584backward -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>1639796325584backward</title>\n",
       "<polygon fill=\"lightgreen\" stroke=\"black\" stroke-width=\"2\" points=\"24.67,-109.5 24.67,-166.5 90.67,-166.5 90.67,-109.5 24.67,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"57.67\" y=\"-154.5\" font-family=\"Menlo\" font-size=\"10.00\">grad=&#45;1.50</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"24.67,-147.5 90.67,-147.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"57.67\" y=\"-135.5\" font-family=\"Menlo\" font-size=\"10.00\">value=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"24.67,-128.5 90.67,-128.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"57.67\" y=\"-116.5\" font-family=\"Menlo\" font-size=\"10.00\">b</text>\n",
       "</g>\n",
       "<!-- 1639796857424backward&#45;&gt;1639796325584backward -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>1639796857424backward&#45;&gt;1639796325584backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M77.68,-218.31C75.99,-212.29 74.24,-205.94 72.67,-200 70.68,-192.45 68.63,-184.35 66.7,-176.6\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"70.05,-175.56 64.27,-166.68 63.26,-177.23 70.05,-175.56\"/>\n",
       "<text text-anchor=\"middle\" x=\"89.17\" y=\"-188.8\" font-family=\"Menlo\" font-size=\"14.00\">&#45;1.50</text>\n",
       "</g>\n",
       "<!-- 1639796319056backward -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>1639796319056backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M120.67,-109.5C120.67,-109.5 162.67,-109.5 162.67,-109.5 168.67,-109.5 174.67,-115.5 174.67,-121.5 174.67,-121.5 174.67,-154.5 174.67,-154.5 174.67,-160.5 168.67,-166.5 162.67,-166.5 162.67,-166.5 120.67,-166.5 120.67,-166.5 114.67,-166.5 108.67,-160.5 108.67,-154.5 108.67,-154.5 108.67,-121.5 108.67,-121.5 108.67,-115.5 114.67,-109.5 120.67,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"141.67\" y=\"-154.5\" font-family=\"Menlo\" font-size=\"10.00\">grad=&#45;1.50</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"108.67,-147.5 174.67,-147.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"141.67\" y=\"-135.5\" font-family=\"Menlo\" font-size=\"10.00\">value=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"108.67,-128.5 174.67,-128.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"141.67\" y=\"-116.5\" font-family=\"Menlo\" font-size=\"10.00\">*</text>\n",
       "</g>\n",
       "<!-- 1639796857424backward&#45;&gt;1639796319056backward -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>1639796857424backward&#45;&gt;1639796319056backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M100.1,-218.44C106.9,-205.44 115.12,-189.74 122.45,-175.74\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"125.57,-177.32 127.11,-166.83 119.37,-174.07 125.57,-177.32\"/>\n",
       "<text text-anchor=\"middle\" x=\"134.17\" y=\"-188.8\" font-family=\"Menlo\" font-size=\"14.00\">&#45;1.50</text>\n",
       "</g>\n",
       "<!-- 1639796830288backward -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>1639796830288backward</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"100.67\" cy=\"-29\" rx=\"31.4\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"100.67\" y=\"-26.5\" font-family=\"Menlo\" font-size=\"10.00\">x1=1.00</text>\n",
       "</g>\n",
       "<!-- 1639808597200backward -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>1639808597200backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M32.67,-545.5C32.67,-545.5 74.67,-545.5 74.67,-545.5 80.67,-545.5 86.67,-551.5 86.67,-557.5 86.67,-557.5 86.67,-590.5 86.67,-590.5 86.67,-596.5 80.67,-602.5 74.67,-602.5 74.67,-602.5 32.67,-602.5 32.67,-602.5 26.67,-602.5 20.67,-596.5 20.67,-590.5 20.67,-590.5 20.67,-557.5 20.67,-557.5 20.67,-551.5 26.67,-545.5 32.67,-545.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"53.67\" y=\"-590.5\" font-family=\"Menlo\" font-size=\"10.00\">grad=1.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"20.67,-583.5 86.67,-583.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"53.67\" y=\"-571.5\" font-family=\"Menlo\" font-size=\"10.00\">value=1.12</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"20.67,-564.5 86.67,-564.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"53.67\" y=\"-552.5\" font-family=\"Menlo\" font-size=\"10.00\">*</text>\n",
       "</g>\n",
       "<!-- 1639808597200backward&#45;&gt;1639808597520backward -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>1639808597200backward&#45;&gt;1639808597520backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M49.55,-545.44C46.61,-525.75 42.74,-499.88 40.23,-483.09\"/>\n",
       "</g>\n",
       "<!-- 1639808592272backward -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>1639808592272backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M105.67,-436.5C105.67,-436.5 147.67,-436.5 147.67,-436.5 153.67,-436.5 159.67,-442.5 159.67,-448.5 159.67,-448.5 159.67,-481.5 159.67,-481.5 159.67,-487.5 153.67,-493.5 147.67,-493.5 147.67,-493.5 105.67,-493.5 105.67,-493.5 99.67,-493.5 93.67,-487.5 93.67,-481.5 93.67,-481.5 93.67,-448.5 93.67,-448.5 93.67,-442.5 99.67,-436.5 105.67,-436.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"126.67\" y=\"-481.5\" font-family=\"Menlo\" font-size=\"10.00\">grad=0.50</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"93.67,-474.5 159.67,-474.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"126.67\" y=\"-462.5\" font-family=\"Menlo\" font-size=\"10.00\">value=2.25</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"93.67,-455.5 159.67,-455.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"126.67\" y=\"-443.5\" font-family=\"Menlo\" font-size=\"10.00\">mse</text>\n",
       "</g>\n",
       "<!-- 1639808597200backward&#45;&gt;1639808592272backward -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>1639808597200backward&#45;&gt;1639808592272backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M72.47,-545.44C81.43,-532.32 92.26,-516.44 101.88,-502.34\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"104.94,-504.07 107.69,-493.83 99.16,-500.12 104.94,-504.07\"/>\n",
       "<text text-anchor=\"middle\" x=\"108.67\" y=\"-515.8\" font-family=\"Menlo\" font-size=\"14.00\">0.50</text>\n",
       "</g>\n",
       "<!-- 1639797063440backward -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>1639797063440backward</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"167.67\" cy=\"-247\" rx=\"31.4\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"167.67\" y=\"-244.5\" font-family=\"Menlo\" font-size=\"10.00\">y1=1.50</text>\n",
       "</g>\n",
       "<!-- 1639808592208backward -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>1639808592208backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M105.67,-327.5C105.67,-327.5 147.67,-327.5 147.67,-327.5 153.67,-327.5 159.67,-333.5 159.67,-339.5 159.67,-339.5 159.67,-372.5 159.67,-372.5 159.67,-378.5 153.67,-384.5 147.67,-384.5 147.67,-384.5 105.67,-384.5 105.67,-384.5 99.67,-384.5 93.67,-378.5 93.67,-372.5 93.67,-372.5 93.67,-339.5 93.67,-339.5 93.67,-333.5 99.67,-327.5 105.67,-327.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"126.67\" y=\"-372.5\" font-family=\"Menlo\" font-size=\"10.00\">grad=1.50</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"93.67,-365.5 159.67,-365.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"126.67\" y=\"-353.5\" font-family=\"Menlo\" font-size=\"10.00\">value=1.50</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"93.67,-346.5 159.67,-346.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"126.67\" y=\"-334.5\" font-family=\"Menlo\" font-size=\"10.00\">&#45;</text>\n",
       "</g>\n",
       "<!-- 1639808592208backward&#45;&gt;1639796857424backward -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>1639808592208backward&#45;&gt;1639796857424backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M115.77,-327.29C113.41,-321.27 110.95,-314.92 108.67,-309 105.7,-301.26 102.55,-292.96 99.56,-285.05\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"102.82,-283.78 96.02,-275.66 96.27,-286.25 102.82,-283.78\"/>\n",
       "<text text-anchor=\"middle\" x=\"125.17\" y=\"-297.8\" font-family=\"Menlo\" font-size=\"14.00\">&#45;1.50</text>\n",
       "</g>\n",
       "<!-- 1639808592208backward&#45;&gt;1639797063440backward -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>1639808592208backward&#45;&gt;1639797063440backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M137.23,-327.44C144.83,-307.63 154.82,-281.55 161.25,-264.78\"/>\n",
       "</g>\n",
       "<!-- 1639796319056backward&#45;&gt;1639796830288backward -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>1639796319056backward&#45;&gt;1639796830288backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M131.12,-109.44C123.52,-89.63 113.53,-63.55 107.1,-46.78\"/>\n",
       "</g>\n",
       "<!-- 1639777488336backward -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>1639777488336backward</title>\n",
       "<polygon fill=\"lightgreen\" stroke=\"black\" stroke-width=\"2\" points=\"149.67,-0.5 149.67,-57.5 215.67,-57.5 215.67,-0.5 149.67,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"182.67\" y=\"-45.5\" font-family=\"Menlo\" font-size=\"10.00\">grad=&#45;1.50</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"149.67,-38.5 215.67,-38.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"182.67\" y=\"-26.5\" font-family=\"Menlo\" font-size=\"10.00\">value=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"149.67,-19.5 215.67,-19.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"182.67\" y=\"-7.5\" font-family=\"Menlo\" font-size=\"10.00\">a</text>\n",
       "</g>\n",
       "<!-- 1639796319056backward&#45;&gt;1639777488336backward -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>1639796319056backward&#45;&gt;1639777488336backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M152.23,-109.44C157.12,-96.69 163,-81.34 168.29,-67.54\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"171.7,-68.42 172.01,-57.83 165.16,-65.92 171.7,-68.42\"/>\n",
       "<text text-anchor=\"middle\" x=\"181.17\" y=\"-79.8\" font-family=\"Menlo\" font-size=\"14.00\">&#45;1.50</text>\n",
       "</g>\n",
       "<!-- 1639808592272backward&#45;&gt;1639808592208backward -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>1639808592272backward&#45;&gt;1639808592208backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M126.67,-436.44C126.67,-423.81 126.67,-408.64 126.67,-394.95\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"130.17,-394.83 126.67,-384.83 123.17,-394.83 130.17,-394.83\"/>\n",
       "<text text-anchor=\"middle\" x=\"140.67\" y=\"-406.8\" font-family=\"Menlo\" font-size=\"14.00\">1.50</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.graphs.Digraph at 0x17dcc342950>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 第一次传播\n",
    "model = Linear()\n",
    "loss = 1/2*mse([model.error(x1,y1)])\n",
    "loss.backward()\n",
    "draw_graph(loss,\"backward\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "(process:20936): Pango-WARNING **: 13:49:58.510: couldn't load font \"Menlo Not-Rotated 10\", falling back to \"Sans Not-Rotated 10\", expect ugly output.\n",
      "\n",
      "(process:20936): Pango-WARNING **: 13:49:58.514: couldn't load font \"Menlo Not-Rotated 14\", falling back to \"Sans Not-Rotated 14\", expect ugly output.\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.50.0 (0)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"235pt\" height=\"611pt\"\n",
       " viewBox=\"0.00 0.00 235.40 611.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 607)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-607 231.4,-607 231.4,4 -4,4\"/>\n",
       "<!-- 1639777303632backward -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>1639777303632backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M51.2,-109.5C51.2,-109.5 93.2,-109.5 93.2,-109.5 99.2,-109.5 105.2,-115.5 105.2,-121.5 105.2,-121.5 105.2,-154.5 105.2,-154.5 105.2,-160.5 99.2,-166.5 93.2,-166.5 93.2,-166.5 51.2,-166.5 51.2,-166.5 45.2,-166.5 39.2,-160.5 39.2,-154.5 39.2,-154.5 39.2,-121.5 39.2,-121.5 39.2,-115.5 45.2,-109.5 51.2,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"72.2\" y=\"-154.5\" font-family=\"Menlo\" font-size=\"10.00\">grad=&#45;4.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"39.2,-147.5 105.2,-147.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"72.2\" y=\"-135.5\" font-family=\"Menlo\" font-size=\"10.00\">value=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"39.2,-128.5 105.2,-128.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"72.2\" y=\"-116.5\" font-family=\"Menlo\" font-size=\"10.00\">*</text>\n",
       "</g>\n",
       "<!-- 1639796422096backward -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>1639796422096backward</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"31.2\" cy=\"-29\" rx=\"31.4\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"31.2\" y=\"-26.5\" font-family=\"Menlo\" font-size=\"10.00\">x2=2.00</text>\n",
       "</g>\n",
       "<!-- 1639777303632backward&#45;&gt;1639796422096backward -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>1639777303632backward&#45;&gt;1639796422096backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M61.64,-109.44C54.05,-89.63 44.06,-63.55 37.63,-46.78\"/>\n",
       "</g>\n",
       "<!-- 1639777488336backward -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>1639777488336backward</title>\n",
       "<polygon fill=\"lightgreen\" stroke=\"black\" stroke-width=\"2\" points=\"80.2,-0.5 80.2,-57.5 146.2,-57.5 146.2,-0.5 80.2,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"113.2\" y=\"-45.5\" font-family=\"Menlo\" font-size=\"10.00\">grad=&#45;9.50</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"80.2,-38.5 146.2,-38.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"113.2\" y=\"-26.5\" font-family=\"Menlo\" font-size=\"10.00\">value=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"80.2,-19.5 146.2,-19.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"113.2\" y=\"-7.5\" font-family=\"Menlo\" font-size=\"10.00\">a</text>\n",
       "</g>\n",
       "<!-- 1639777303632backward&#45;&gt;1639777488336backward -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>1639777303632backward&#45;&gt;1639777488336backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M82.76,-109.44C87.65,-96.69 93.53,-81.34 98.81,-67.54\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"102.22,-68.42 102.53,-57.83 95.69,-65.92 102.22,-68.42\"/>\n",
       "<text text-anchor=\"middle\" x=\"111.7\" y=\"-79.8\" font-family=\"Menlo\" font-size=\"14.00\">&#45;8.00</text>\n",
       "</g>\n",
       "<!-- 1639808511696backward -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>1639808511696backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M93.2,-218.5C93.2,-218.5 135.2,-218.5 135.2,-218.5 141.2,-218.5 147.2,-224.5 147.2,-230.5 147.2,-230.5 147.2,-263.5 147.2,-263.5 147.2,-269.5 141.2,-275.5 135.2,-275.5 135.2,-275.5 93.2,-275.5 93.2,-275.5 87.2,-275.5 81.2,-269.5 81.2,-263.5 81.2,-263.5 81.2,-230.5 81.2,-230.5 81.2,-224.5 87.2,-218.5 93.2,-218.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"114.2\" y=\"-263.5\" font-family=\"Menlo\" font-size=\"10.00\">grad=&#45;4.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"81.2,-256.5 147.2,-256.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"114.2\" y=\"-244.5\" font-family=\"Menlo\" font-size=\"10.00\">value=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"81.2,-237.5 147.2,-237.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"114.2\" y=\"-225.5\" font-family=\"Menlo\" font-size=\"10.00\">+</text>\n",
       "</g>\n",
       "<!-- 1639808511696backward&#45;&gt;1639777303632backward -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>1639808511696backward&#45;&gt;1639777303632backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M103.38,-218.44C98.38,-205.69 92.35,-190.34 86.94,-176.54\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"90.04,-174.86 83.12,-166.83 83.52,-177.42 90.04,-174.86\"/>\n",
       "<text text-anchor=\"middle\" x=\"112.7\" y=\"-188.8\" font-family=\"Menlo\" font-size=\"14.00\">&#45;4.00</text>\n",
       "</g>\n",
       "<!-- 1639796325584backward -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>1639796325584backward</title>\n",
       "<polygon fill=\"lightgreen\" stroke=\"black\" stroke-width=\"2\" points=\"123.2,-109.5 123.2,-166.5 189.2,-166.5 189.2,-109.5 123.2,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"156.2\" y=\"-154.5\" font-family=\"Menlo\" font-size=\"10.00\">grad=&#45;5.50</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"123.2,-147.5 189.2,-147.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"156.2\" y=\"-135.5\" font-family=\"Menlo\" font-size=\"10.00\">value=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"123.2,-128.5 189.2,-128.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"156.2\" y=\"-116.5\" font-family=\"Menlo\" font-size=\"10.00\">b</text>\n",
       "</g>\n",
       "<!-- 1639808511696backward&#45;&gt;1639796325584backward -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>1639808511696backward&#45;&gt;1639796325584backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M125.02,-218.44C130.02,-205.69 136.05,-190.34 141.46,-176.54\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"144.88,-177.42 145.27,-166.83 138.36,-174.86 144.88,-177.42\"/>\n",
       "<text text-anchor=\"middle\" x=\"154.7\" y=\"-188.8\" font-family=\"Menlo\" font-size=\"14.00\">&#45;4.00</text>\n",
       "</g>\n",
       "<!-- 1639796323536backward -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>1639796323536backward</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"196.2\" cy=\"-247\" rx=\"31.4\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"196.2\" y=\"-244.5\" font-family=\"Menlo\" font-size=\"10.00\">y2=4.00</text>\n",
       "</g>\n",
       "<!-- 1639808509200backward -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>1639808509200backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M134.2,-327.5C134.2,-327.5 176.2,-327.5 176.2,-327.5 182.2,-327.5 188.2,-333.5 188.2,-339.5 188.2,-339.5 188.2,-372.5 188.2,-372.5 188.2,-378.5 182.2,-384.5 176.2,-384.5 176.2,-384.5 134.2,-384.5 134.2,-384.5 128.2,-384.5 122.2,-378.5 122.2,-372.5 122.2,-372.5 122.2,-339.5 122.2,-339.5 122.2,-333.5 128.2,-327.5 134.2,-327.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"155.2\" y=\"-372.5\" font-family=\"Menlo\" font-size=\"10.00\">grad=4.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"122.2,-365.5 188.2,-365.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155.2\" y=\"-353.5\" font-family=\"Menlo\" font-size=\"10.00\">value=4.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"122.2,-346.5 188.2,-346.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155.2\" y=\"-334.5\" font-family=\"Menlo\" font-size=\"10.00\">&#45;</text>\n",
       "</g>\n",
       "<!-- 1639808509200backward&#45;&gt;1639808511696backward -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>1639808509200backward&#45;&gt;1639808511696backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M144.64,-327.44C139.75,-314.69 133.87,-299.34 128.58,-285.54\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"131.71,-283.92 124.86,-275.83 125.17,-286.42 131.71,-283.92\"/>\n",
       "<text text-anchor=\"middle\" x=\"153.7\" y=\"-297.8\" font-family=\"Menlo\" font-size=\"14.00\">&#45;4.00</text>\n",
       "</g>\n",
       "<!-- 1639808509200backward&#45;&gt;1639796323536backward -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>1639808509200backward&#45;&gt;1639796323536backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M166.86,-327.35C169.33,-321.32 171.89,-314.96 174.2,-309 179.92,-294.19 186.02,-277.19 190.33,-264.91\"/>\n",
       "</g>\n",
       "<!-- 1639808489232backward -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>1639808489232backward</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"64.2\" cy=\"-465\" rx=\"37.85\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"64.2\" y=\"-462.5\" font-family=\"Menlo\" font-size=\"10.00\">input=0.50</text>\n",
       "</g>\n",
       "<!-- 1639808494928backward -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>1639808494928backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M59.2,-545.5C59.2,-545.5 101.2,-545.5 101.2,-545.5 107.2,-545.5 113.2,-551.5 113.2,-557.5 113.2,-557.5 113.2,-590.5 113.2,-590.5 113.2,-596.5 107.2,-602.5 101.2,-602.5 101.2,-602.5 59.2,-602.5 59.2,-602.5 53.2,-602.5 47.2,-596.5 47.2,-590.5 47.2,-590.5 47.2,-557.5 47.2,-557.5 47.2,-551.5 53.2,-545.5 59.2,-545.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"80.2\" y=\"-590.5\" font-family=\"Menlo\" font-size=\"10.00\">grad=1.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"47.2,-583.5 113.2,-583.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"80.2\" y=\"-571.5\" font-family=\"Menlo\" font-size=\"10.00\">value=8.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"47.2,-564.5 113.2,-564.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"80.2\" y=\"-552.5\" font-family=\"Menlo\" font-size=\"10.00\">*</text>\n",
       "</g>\n",
       "<!-- 1639808494928backward&#45;&gt;1639808489232backward -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>1639808494928backward&#45;&gt;1639808489232backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M76.08,-545.44C73.13,-525.75 69.26,-499.88 66.76,-483.09\"/>\n",
       "</g>\n",
       "<!-- 1639808485840backward -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>1639808485840backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M131.7,-436.5C131.7,-436.5 178.7,-436.5 178.7,-436.5 184.7,-436.5 190.7,-442.5 190.7,-448.5 190.7,-448.5 190.7,-481.5 190.7,-481.5 190.7,-487.5 184.7,-493.5 178.7,-493.5 178.7,-493.5 131.7,-493.5 131.7,-493.5 125.7,-493.5 119.7,-487.5 119.7,-481.5 119.7,-481.5 119.7,-448.5 119.7,-448.5 119.7,-442.5 125.7,-436.5 131.7,-436.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"155.2\" y=\"-481.5\" font-family=\"Menlo\" font-size=\"10.00\">grad=0.50</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"119.7,-474.5 190.7,-474.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155.2\" y=\"-462.5\" font-family=\"Menlo\" font-size=\"10.00\">value=16.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"119.7,-455.5 190.7,-455.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155.2\" y=\"-443.5\" font-family=\"Menlo\" font-size=\"10.00\">mse</text>\n",
       "</g>\n",
       "<!-- 1639808494928backward&#45;&gt;1639808485840backward -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>1639808494928backward&#45;&gt;1639808485840backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M99.51,-545.44C108.71,-532.32 119.84,-516.44 129.73,-502.34\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"132.82,-504.03 135.69,-493.83 127.08,-500.01 132.82,-504.03\"/>\n",
       "<text text-anchor=\"middle\" x=\"136.2\" y=\"-515.8\" font-family=\"Menlo\" font-size=\"14.00\">0.50</text>\n",
       "</g>\n",
       "<!-- 1639808485840backward&#45;&gt;1639808509200backward -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>1639808485840backward&#45;&gt;1639808509200backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M155.2,-436.44C155.2,-423.81 155.2,-408.64 155.2,-394.95\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"158.7,-394.83 155.2,-384.83 151.7,-394.83 158.7,-394.83\"/>\n",
       "<text text-anchor=\"middle\" x=\"169.2\" y=\"-406.8\" font-family=\"Menlo\" font-size=\"14.00\">4.00</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.graphs.Digraph at 0x17dcc33bb90>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 第二次传播(梯度积累)\n",
    "loss = 1/2*mse([model.error(x2,y2)])\n",
    "loss.backward()\n",
    "draw_graph(loss,\"backward\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y = 0.96 * x + -0.24\n",
      "y = 4.48 * x + 2.03\n",
      "y = 5.26 * x + 2.99\n",
      "y = 6.29 * x + 3.36\n",
      "y = 7.43 * x + 4.01\n",
      "y = 7.64 * x + 4.23\n",
      "y = 8.23 * x + 4.25\n",
      "y = 8.85 * x + 4.75\n",
      "y = 8.92 * x + 4.74\n",
      "y = 9.26 * x + 4.87\n",
      "y = 9.43 * x + 5.00\n",
      "y = 9.48 * x + 4.95\n",
      "y = 9.64 * x + 4.96\n",
      "y = 9.69 * x + 5.00\n",
      "y = 9.76 * x + 5.02\n",
      "y = 9.83 * x + 5.05\n",
      "y = 9.84 * x + 5.03\n",
      "y = 9.88 * x + 5.04\n",
      "y = 9.92 * x + 5.03\n",
      "y = 9.91 * x + 5.01\n"
     ]
    }
   ],
   "source": [
    "model = Linear()\n",
    "\n",
    "batch_size = 32\n",
    "lr = 0.1\n",
    "# 梯度累计次数\n",
    "gradient_accu_iter = 4\n",
    "# 小批量数据量\n",
    "micro_size = int(batch_size/gradient_accu_iter)\n",
    "\n",
    "for t in range(20 * gradient_accu_iter):\n",
    "    ix = (t * batch_size) % len(x)\n",
    "    xx = x[ix:ix+batch_size]\n",
    "    yy = y[ix:ix+batch_size]\n",
    "    \n",
    "    # 条件损失的权重\n",
    "    loss = 1/gradient_accu_iter*mse([model.error(_x,_y) for _x,_y in zip(xx,yy)])\n",
    "    loss.backward()\n",
    "    if (t+1) % gradient_accu_iter == 0:\n",
    "        model.a -= lr * model.a.grad\n",
    "        model.b -= lr * model.b.grad\n",
    "        model.a.grad = 0 \n",
    "        model.b.grad = 0\n",
    "        print(model.string())\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nvgpu",
   "language": "python",
   "name": "nvgpu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
